Upload folder using huggingface_hub

mistral7b_kto_kl-beta0.01/README.md

@@ -0,0 +1,64 @@
+---
+library_name: peft
+license: other
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- base_model:adapter:mistralai/Mistral-7B-v0.1
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: mistral7b_kto_kl
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral7b_kto_kl
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the shp_kto_train dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 64
+- optimizer: Use adamw_torch_fused with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.17.1
+- Transformers 4.57.1
+- Pytorch 2.10.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2

mistral7b_kto_kl-beta0.01/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

mistral7b_kto_kl-beta0.01/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2826bd7424abff25d73c708e9451220de8b86d961c4b70b3cae81a4b31a632f
+size 42002584

mistral7b_kto_kl-beta0.01/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.1861866257511875e+18,
+    "train_loss": 0.5003062401499067,
+    "train_runtime": 3152.1109,
+    "train_samples_per_second": 31.725,
+    "train_steps_per_second": 0.062
+}

mistral7b_kto_kl-beta0.01/chat_template.jinja

@@ -0,0 +1,3 @@
+{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 and system_message is defined %}{% set content = system_message + '
+
+' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + '[/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content + '</s>' }}{% endif %}{% endfor %}

mistral7b_kto_kl-beta0.01/checkpoint-196/README.md

@@ -0,0 +1,208 @@
+---
+base_model: mistralai/Mistral-7B-v0.1
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:mistralai/Mistral-7B-v0.1
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

mistral7b_kto_kl-beta0.01/checkpoint-196/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "down_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

mistral7b_kto_kl-beta0.01/checkpoint-196/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2826bd7424abff25d73c708e9451220de8b86d961c4b70b3cae81a4b31a632f
+size 42002584

mistral7b_kto_kl-beta0.01/checkpoint-196/chat_template.jinja

@@ -0,0 +1,3 @@
+{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 and system_message is defined %}{% set content = system_message + '
+
+' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + '[/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content + '</s>' }}{% endif %}{% endfor %}

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53e5667d59e4d850d55247aa7e9119218ee2c16a5ece9f941449f1a5450cb1bc
+size 31467333

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fce3e2b4c27ded4e767dc201fd31470d7a076495f84b994cc318a521bad4891
+size 31467333

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8795099015c865f54b5d709f1b32dc82ac5665ad59e47dc8a8e796b0dcd6680c
+size 31467333

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00c2fd3a983f50dca9ddcb2b017ad741b67778268e1207d7f5f0296530173fc1
+size 31467397

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dee78feaf705a1fff18755c11fea6ec1afe55d48397e56d0da8e8a3f5995d249
+size 31467397

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945ce229c227a34d9767d1ea8109e6560043e960e6c44980b1afda1be35412f0
+size 31467397

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52c06759b394677d92106e095e15d245022f41d3297adc0be079eb3f7f8a278a
+size 31467397

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79cdc78d39c657049e5a44af16fff190630f83f2b9d7cd7df9854615466d9c5d
+size 31467397

mistral7b_kto_kl-beta0.01/checkpoint-196/global_step196/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6a29afdd782ee66007f004d8f29169ac89e17b99811235e3a0ac81c9f974916
+size 42288409

mistral7b_kto_kl-beta0.01/checkpoint-196/latest

@@ -0,0 +1 @@
+global_step196

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7efbc3b39e557ed047a255851058c3d62216b6bc5794c19289879cf099f52035
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce29a8767a7d907dd24987aa2c3e654d4317f3042fbc13b5b72cadb46d43311a
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a48db011646b4e9a867bf12f4a233cad5dfbfe309686f8996c250196d3783a
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9562ee822472a4f01dcd6349ab3d1ef42a48915fe3b92e843a0c37db53c8421
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d2767d83c3bf27f12db022b0632e2c4f8c164274ba75e380cf18f9d5f21819
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76816358d4e5db8149d60d55234db658d67a13c0c1ce05d7404cf7125a676a5c
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1562e7520c977d178183d641f70abcf3f57da2489938756cfbebf9b6e6c1a9fd
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b6cabaed045c5398cd1b732f7ec48bd363f3b43cd24e0e70e641a42bd00c28
+size 16389

mistral7b_kto_kl-beta0.01/checkpoint-196/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83820952e8eb59075696a2d6e1f7ef27d70c5576cb2781806606f38f86685d3e
+size 1401

mistral7b_kto_kl-beta0.01/checkpoint-196/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mistral7b_kto_kl-beta0.01/checkpoint-196/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

mistral7b_kto_kl-beta0.01/checkpoint-196/tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

mistral7b_kto_kl-beta0.01/checkpoint-196/trainer_state.json

@@ -0,0 +1,1004 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 196,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.025591810620601407,
+      "grad_norm": 0.10970883816480637,
+      "kl": 0.00977869238704443,
+      "learning_rate": 4e-08,
+      "logits/chosen": -59719296.0,
+      "logits/rejected": -50418438.0952381,
+      "logps/chosen": -439.02749794407896,
+      "logps/rejected": -198.41131882440476,
+      "loss": 0.5002,
+      "loss/base_kto": 0.49997520446777344,
+      "loss/total_with_kl": 0.5001660585403442,
+      "metrics/advantage_var": 0.03817559406161308,
+      "regularization/advantage_var": 0.03817559406161308,
+      "regularization/kl": 0.00019087796681560576,
+      "rewards/chosen": 0.00018095341511070728,
+      "rewards/margins": 0.0002000417693385056,
+      "rewards/rejected": -1.9088354227798326e-05,
+      "step": 5
+    },
+    {
+      "epoch": 0.05118362124120281,
+      "grad_norm": 0.09480192512273788,
+      "kl": 0.0236990787088871,
+      "learning_rate": 9e-08,
+      "logits/chosen": -59294786.49350649,
+      "logits/rejected": -50011718.93975904,
+      "logps/chosen": -429.914163961039,
+      "logps/rejected": -192.84071442018072,
+      "loss": 0.5003,
+      "loss/base_kto": 0.500006377696991,
+      "loss/total_with_kl": 0.5003525614738464,
+      "metrics/advantage_var": 0.06924548000097275,
+      "regularization/advantage_var": 0.06924548000097275,
+      "regularization/kl": 0.00034622737439349294,
+      "rewards/chosen": 0.00024344235078080907,
+      "rewards/margins": -2.866059493904772e-05,
+      "rewards/rejected": 0.0002721029457198568,
+      "stability/repetition_rate_mean": 0.3590347170829773,
+      "stability/response_length_mean": 46.125,
+      "stability/response_length_std": 27.231481552124023,
+      "stability/response_length_var": 741.5535888671875,
+      "stability/token_entropy_mean": 3.33642578125,
+      "step": 10
+    },
+    {
+      "epoch": 0.07677543186180422,
+      "grad_norm": 0.11248399317264557,
+      "kl": 0.014778117649257183,
+      "learning_rate": 1.4e-07,
+      "logits/chosen": -56242337.35757576,
+      "logits/rejected": -51928440.56774194,
+      "logps/chosen": -404.1227272727273,
+      "logps/rejected": -200.6803931451613,
+      "loss": 0.5003,
+      "loss/base_kto": 0.500006914138794,
+      "loss/total_with_kl": 0.5003097653388977,
+      "metrics/advantage_var": 0.06057020649313927,
+      "regularization/advantage_var": 0.06057020649313927,
+      "regularization/kl": 0.000302851025480777,
+      "rewards/chosen": 8.117372565197222e-05,
+      "rewards/margins": -5.552637866923071e-05,
+      "rewards/rejected": 0.00013670010432120292,
+      "stability/repetition_rate_mean": 0.2929687201976776,
+      "stability/response_length_mean": 29.375,
+      "stability/response_length_std": 29.92341423034668,
+      "stability/response_length_var": 895.4107055664062,
+      "stability/token_entropy_mean": 3.7041015625,
+      "step": 15
+    },
+    {
+      "epoch": 0.10236724248240563,
+      "grad_norm": 0.11460169404745102,
+      "kl": 0.0199257992208004,
+      "learning_rate": 1.8999999999999998e-07,
+      "logits/chosen": -51660729.6,
+      "logits/rejected": -48058240.0,
+      "logps/chosen": -343.7023193359375,
+      "logps/rejected": -225.6357421875,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000439286231995,
+      "loss/total_with_kl": 0.5003700852394104,
+      "metrics/advantage_var": 0.06522905826568604,
+      "regularization/advantage_var": 0.06522905826568604,
+      "regularization/kl": 0.00032614529482088983,
+      "rewards/chosen": -5.6106073316186666e-05,
+      "rewards/margins": -0.0003321924596093595,
+      "rewards/rejected": 0.00027608638629317284,
+      "stability/repetition_rate_mean": 0.20810391008853912,
+      "stability/response_length_mean": 37.25,
+      "stability/response_length_std": 29.040611267089844,
+      "stability/response_length_var": 843.3571166992188,
+      "stability/token_entropy_mean": 4.49609375,
+      "step": 20
+    },
+    {
+      "epoch": 0.12795905310300704,
+      "grad_norm": 0.11199703812599182,
+      "kl": 0.01614060439169407,
+      "learning_rate": 2.4e-07,
+      "logits/chosen": -56484991.11724138,
+      "logits/rejected": -52026502.58285714,
+      "logps/chosen": -378.48811961206894,
+      "logps/rejected": -202.79370535714287,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999929368495941,
+      "loss/total_with_kl": 0.5002965927124023,
+      "metrics/advantage_var": 0.06073552370071411,
+      "regularization/advantage_var": 0.06073552370071411,
+      "regularization/kl": 0.0003036776324734092,
+      "rewards/chosen": 3.336018015598429e-05,
+      "rewards/margins": 4.192581996688702e-05,
+      "rewards/rejected": -8.565639810902731e-06,
+      "stability/repetition_rate_mean": 0.18344351649284363,
+      "stability/response_length_mean": 50.25,
+      "stability/response_length_std": 25.549671173095703,
+      "stability/response_length_var": 652.7857055664062,
+      "stability/token_entropy_mean": 4.5546875,
+      "step": 25
+    },
+    {
+      "epoch": 0.15355086372360843,
+      "grad_norm": 0.10699352622032166,
+      "kl": 0.016464022919535637,
+      "learning_rate": 2.9e-07,
+      "logits/chosen": -57725168.536912754,
+      "logits/rejected": -51507990.456140354,
+      "logps/chosen": -468.56894924496646,
+      "logps/rejected": -198.23707054093566,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999980032444,
+      "loss/total_with_kl": 0.5003641247749329,
+      "metrics/advantage_var": 0.07323050498962402,
+      "regularization/advantage_var": 0.07323050498962402,
+      "regularization/kl": 0.0003661524970084429,
+      "rewards/chosen": 0.00012335383752048415,
+      "rewards/margins": 1.539026261185396e-05,
+      "rewards/rejected": 0.00010796357490863019,
+      "stability/repetition_rate_mean": 0.1583891361951828,
+      "stability/response_length_mean": 43.0,
+      "stability/response_length_std": 29.602123260498047,
+      "stability/response_length_var": 876.2857055664062,
+      "stability/token_entropy_mean": 4.66015625,
+      "step": 30
+    },
+    {
+      "epoch": 0.17914267434420986,
+      "grad_norm": 0.21138133108615875,
+      "kl": 0.019741442054510117,
+      "learning_rate": 3.4000000000000003e-07,
+      "logits/chosen": -57637447.29113924,
+      "logits/rejected": -53896343.7037037,
+      "logps/chosen": -385.9375,
+      "logps/rejected": -195.11993634259258,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000119805335999,
+      "loss/total_with_kl": 0.5003029108047485,
+      "metrics/advantage_var": 0.058179568499326706,
+      "regularization/advantage_var": 0.058179568499326706,
+      "regularization/kl": 0.0002908978785853833,
+      "rewards/chosen": -9.798232607449158e-06,
+      "rewards/margins": -7.508864769210554e-05,
+      "rewards/rejected": 6.529041508465638e-05,
+      "stability/repetition_rate_mean": 0.33708456158638,
+      "stability/response_length_mean": 53.375,
+      "stability/response_length_std": 21.14533042907715,
+      "stability/response_length_var": 447.125,
+      "stability/token_entropy_mean": 5.330078125,
+      "step": 35
+    },
+    {
+      "epoch": 0.20473448496481125,
+      "grad_norm": 0.1126021295785904,
+      "kl": 0.021908169612288475,
+      "learning_rate": 3.8999999999999997e-07,
+      "logits/chosen": -56632742.4,
+      "logits/rejected": -50238966.4,
+      "logps/chosen": -421.8955078125,
+      "logps/rejected": -219.26396484375,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000482797622681,
+      "loss/total_with_kl": 0.5004304647445679,
+      "metrics/advantage_var": 0.07644052803516388,
+      "regularization/advantage_var": 0.07644052803516388,
+      "regularization/kl": 0.00038220261922106147,
+      "rewards/chosen": 7.171009201556444e-05,
+      "rewards/margins": -0.000381213310174644,
+      "rewards/rejected": 0.0004529234021902084,
+      "stability/repetition_rate_mean": 0.16694079339504242,
+      "stability/response_length_mean": 35.0,
+      "stability/response_length_std": 31.518701553344727,
+      "stability/response_length_var": 993.4285888671875,
+      "stability/token_entropy_mean": 4.89453125,
+      "step": 40
+    },
+    {
+      "epoch": 0.23032629558541268,
+      "grad_norm": 0.11807987093925476,
+      "kl": 0.034686531871557236,
+      "learning_rate": 4.3999999999999997e-07,
+      "logits/chosen": -52694314.95302013,
+      "logits/rejected": -50773680.65497076,
+      "logps/chosen": -352.3178743708054,
+      "logps/rejected": -243.85603983918128,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49999910593032837,
+      "loss/total_with_kl": 0.5003305673599243,
+      "metrics/advantage_var": 0.06628922373056412,
+      "regularization/advantage_var": 0.06628922373056412,
+      "regularization/kl": 0.000331446120981127,
+      "rewards/chosen": 0.00010591877886912967,
+      "rewards/margins": -7.539941105788222e-05,
+      "rewards/rejected": 0.0001813181899270119,
+      "stability/repetition_rate_mean": 0.0974486693739891,
+      "stability/response_length_mean": 28.875,
+      "stability/response_length_std": 29.87324333190918,
+      "stability/response_length_var": 892.4107055664062,
+      "stability/token_entropy_mean": 4.8203125,
+      "step": 45
+    },
+    {
+      "epoch": 0.2559181062060141,
+      "grad_norm": 0.08920294046401978,
+      "kl": 0.036549247801303864,
+      "learning_rate": 4.9e-07,
+      "logits/chosen": -54048891.70469799,
+      "logits/rejected": -48298175.62573099,
+      "logps/chosen": -375.1624370805369,
+      "logps/rejected": -180.2200749269006,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999710023403168,
+      "loss/total_with_kl": 0.500243604183197,
+      "metrics/advantage_var": 0.0545235350728035,
+      "regularization/advantage_var": 0.0545235350728035,
+      "regularization/kl": 0.00027261764626018703,
+      "rewards/chosen": 0.0003667769816097797,
+      "rewards/margins": 0.00016145475293268815,
+      "rewards/rejected": 0.00020532222867709156,
+      "stability/repetition_rate_mean": 0.3931290805339813,
+      "stability/response_length_mean": 54.625,
+      "stability/response_length_std": 16.044023513793945,
+      "stability/response_length_var": 257.41070556640625,
+      "stability/token_entropy_mean": 3.8291015625,
+      "step": 50
+    },
+    {
+      "epoch": 0.28150991682661547,
+      "grad_norm": 0.12241534143686295,
+      "kl": 0.03960302472114563,
+      "learning_rate": 5.4e-07,
+      "logits/chosen": -56953518.9068323,
+      "logits/rejected": -49572674.012578614,
+      "logps/chosen": -344.4889848602484,
+      "logps/rejected": -230.42143278301887,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000047087669373,
+      "loss/total_with_kl": 0.500251829624176,
+      "metrics/advantage_var": 0.04942348971962929,
+      "regularization/advantage_var": 0.04942348971962929,
+      "regularization/kl": 0.0002471174520906061,
+      "rewards/chosen": 0.0002302746780170417,
+      "rewards/margins": -3.589604861513053e-05,
+      "rewards/rejected": 0.0002661707266321722,
+      "stability/repetition_rate_mean": 0.07375919073820114,
+      "stability/response_length_mean": 18.625,
+      "stability/response_length_std": 28.560400009155273,
+      "stability/response_length_var": 815.6964111328125,
+      "stability/token_entropy_mean": 4.59375,
+      "step": 55
+    },
+    {
+      "epoch": 0.30710172744721687,
+      "grad_norm": 0.09351801872253418,
+      "kl": 0.03357061371207237,
+      "learning_rate": 5.9e-07,
+      "logits/chosen": -52676881.06666667,
+      "logits/rejected": -47836245.88387097,
+      "logps/chosen": -359.6825994318182,
+      "logps/rejected": -178.19910534274194,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999830424785614,
+      "loss/total_with_kl": 0.5002629160881042,
+      "metrics/advantage_var": 0.05598118528723717,
+      "regularization/advantage_var": 0.05598118528723717,
+      "regularization/kl": 0.0002799059438984841,
+      "rewards/chosen": 0.0003821046966494936,
+      "rewards/margins": 0.00010400982910470412,
+      "rewards/rejected": 0.0002780948675447895,
+      "stability/repetition_rate_mean": 0.21421611309051514,
+      "stability/response_length_mean": 51.125,
+      "stability/response_length_std": 21.90523338317871,
+      "stability/response_length_var": 479.83929443359375,
+      "stability/token_entropy_mean": 5.30078125,
+      "step": 60
+    },
+    {
+      "epoch": 0.3326935380678183,
+      "grad_norm": 0.10334884375333786,
+      "kl": 0.045144613832235336,
+      "learning_rate": 6.4e-07,
+      "logits/chosen": -57574505.70322581,
+      "logits/rejected": -47286880.193939395,
+      "logps/chosen": -381.9140877016129,
+      "logps/rejected": -183.5405303030303,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000074505805969,
+      "loss/total_with_kl": 0.5003567934036255,
+      "metrics/advantage_var": 0.06986110657453537,
+      "regularization/advantage_var": 0.06986110657453537,
+      "regularization/kl": 0.0003493055410217494,
+      "rewards/chosen": 0.0003270598428864633,
+      "rewards/margins": -9.128375530592506e-05,
+      "rewards/rejected": 0.00041834359819238837,
+      "stability/repetition_rate_mean": 0.1857219785451889,
+      "stability/response_length_mean": 20.625,
+      "stability/response_length_std": 27.176342010498047,
+      "stability/response_length_var": 738.5535888671875,
+      "stability/token_entropy_mean": 5.548828125,
+      "step": 65
+    },
+    {
+      "epoch": 0.3582853486884197,
+      "grad_norm": 0.08497199416160583,
+      "kl": 0.04333776980638504,
+      "learning_rate": 6.9e-07,
+      "logits/chosen": -52786750.76129032,
+      "logits/rejected": -49049150.06060606,
+      "logps/chosen": -379.0890120967742,
+      "logps/rejected": -220.40785984848486,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49996206164360046,
+      "loss/total_with_kl": 0.5003085732460022,
+      "metrics/advantage_var": 0.06930377334356308,
+      "regularization/advantage_var": 0.06930377334356308,
+      "regularization/kl": 0.00034651882015168667,
+      "rewards/chosen": 0.00039446188076849907,
+      "rewards/margins": 0.00025153014579476496,
+      "rewards/rejected": 0.00014293173497373409,
+      "stability/repetition_rate_mean": 0.185546875,
+      "stability/response_length_mean": 33.875,
+      "stability/response_length_std": 32.36151885986328,
+      "stability/response_length_var": 1047.267822265625,
+      "stability/token_entropy_mean": 5.71875,
+      "step": 70
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.10053345561027527,
+      "kl": 0.03558317944407463,
+      "learning_rate": 7.4e-07,
+      "logits/chosen": -49513603.45945946,
+      "logits/rejected": -47910197.58139535,
+      "logps/chosen": -314.19855891047297,
+      "logps/rejected": -228.85667242005815,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999457895755768,
+      "loss/total_with_kl": 0.5002729892730713,
+      "metrics/advantage_var": 0.06544167548418045,
+      "regularization/advantage_var": 0.06544167548418045,
+      "regularization/kl": 0.00032720837043598294,
+      "rewards/chosen": 0.000722566871224223,
+      "rewards/margins": 0.00044078929180022984,
+      "rewards/rejected": 0.00028177757942399314,
+      "stability/repetition_rate_mean": 0.08548677712678909,
+      "stability/response_length_mean": 22.75,
+      "stability/response_length_std": 31.07479476928711,
+      "stability/response_length_var": 965.6428833007812,
+      "stability/token_entropy_mean": 2.1484375,
+      "step": 75
+    },
+    {
+      "epoch": 0.4094689699296225,
+      "grad_norm": 0.09504130482673645,
+      "kl": 0.04469752684235573,
+      "learning_rate": 7.9e-07,
+      "logits/chosen": -58642201.430463575,
+      "logits/rejected": -53484331.92899408,
+      "logps/chosen": -403.9830298013245,
+      "logps/rejected": -201.04558062130178,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49995794892311096,
+      "loss/total_with_kl": 0.5002338290214539,
+      "metrics/advantage_var": 0.05516833811998367,
+      "regularization/advantage_var": 0.05516833811998367,
+      "regularization/kl": 0.0002758416812866926,
+      "rewards/chosen": 0.0004708408717288087,
+      "rewards/margins": 0.00036850688271240556,
+      "rewards/rejected": 0.00010233398901640311,
+      "stability/repetition_rate_mean": 0.18572917580604553,
+      "stability/response_length_mean": 40.25,
+      "stability/response_length_std": 29.421810150146484,
+      "stability/response_length_var": 865.6428833007812,
+      "stability/token_entropy_mean": 5.23828125,
+      "step": 80
+    },
+    {
+      "epoch": 0.4350607805502239,
+      "grad_norm": 0.10743357986211777,
+      "kl": 0.038033999502658844,
+      "learning_rate": 8.399999999999999e-07,
+      "logits/chosen": -51504801.85806452,
+      "logits/rejected": -47172201.5030303,
+      "logps/chosen": -358.2844254032258,
+      "logps/rejected": -230.40402462121213,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999658167362213,
+      "loss/total_with_kl": 0.5002753138542175,
+      "metrics/advantage_var": 0.061900604516267776,
+      "regularization/advantage_var": 0.061900604516267776,
+      "regularization/kl": 0.00030950302607379854,
+      "rewards/chosen": 0.00048093819810498146,
+      "rewards/margins": 0.00027335531434239063,
+      "rewards/rejected": 0.00020758288376259082,
+      "stability/repetition_rate_mean": 0.1350446343421936,
+      "stability/response_length_mean": 28.0,
+      "stability/response_length_std": 31.181495666503906,
+      "stability/response_length_var": 972.2857055664062,
+      "stability/token_entropy_mean": 4.95703125,
+      "step": 85
+    },
+    {
+      "epoch": 0.46065259117082535,
+      "grad_norm": 0.10124877840280533,
+      "kl": 0.04362564906477928,
+      "learning_rate": 8.9e-07,
+      "logits/chosen": -58452775.384615384,
+      "logits/rejected": -50602352.3902439,
+      "logps/chosen": -380.7494491185897,
+      "logps/rejected": -187.19999285442074,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49998241662979126,
+      "loss/total_with_kl": 0.5003246665000916,
+      "metrics/advantage_var": 0.06845911592245102,
+      "regularization/advantage_var": 0.06845911592245102,
+      "regularization/kl": 0.0003422955924179405,
+      "rewards/chosen": 0.00039930832691681693,
+      "rewards/margins": 0.0001312548590543272,
+      "rewards/rejected": 0.0002680534678624897,
+      "stability/repetition_rate_mean": 0.2426609843969345,
+      "stability/response_length_mean": 26.625,
+      "stability/response_length_std": 27.79998779296875,
+      "stability/response_length_var": 772.8392944335938,
+      "stability/token_entropy_mean": 4.29296875,
+      "step": 90
+    },
+    {
+      "epoch": 0.48624440179142675,
+      "grad_norm": 0.10090715438127518,
+      "kl": 0.030433058738708496,
+      "learning_rate": 9.399999999999999e-07,
+      "logits/chosen": -50427842.43037975,
+      "logits/rejected": -49013747.358024694,
+      "logps/chosen": -314.9666732594937,
+      "logps/rejected": -207.13416280864197,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49998149275779724,
+      "loss/total_with_kl": 0.5002912878990173,
+      "metrics/advantage_var": 0.06193568930029869,
+      "regularization/advantage_var": 0.06193568930029869,
+      "regularization/kl": 0.00030967846396379173,
+      "rewards/chosen": 0.00026683563578732404,
+      "rewards/margins": 0.00017124155932459015,
+      "rewards/rejected": 9.55940764627339e-05,
+      "stability/repetition_rate_mean": 0.20546108484268188,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 31.1264705657959,
+      "stability/response_length_var": 968.8571166992188,
+      "stability/token_entropy_mean": 3.6875,
+      "step": 95
+    },
+    {
+      "epoch": 0.5118362124120281,
+      "grad_norm": 0.11534037441015244,
+      "kl": 0.03825628012418747,
+      "learning_rate": 9.9e-07,
+      "logits/chosen": -54771206.91891892,
+      "logits/rejected": -50969665.488372095,
+      "logps/chosen": -359.78301836993245,
+      "logps/rejected": -216.57428688226744,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998970925807953,
+      "loss/total_with_kl": 0.5002375841140747,
+      "metrics/advantage_var": 0.06808135658502579,
+      "regularization/advantage_var": 0.06808135658502579,
+      "regularization/kl": 0.00034040678292512894,
+      "rewards/chosen": 0.0008716205688747199,
+      "rewards/margins": 0.000825567181953267,
+      "rewards/rejected": 4.605338692145292e-05,
+      "stability/repetition_rate_mean": 0.23228970170021057,
+      "stability/response_length_mean": 37.875,
+      "stability/response_length_std": 30.248672485351562,
+      "stability/response_length_var": 914.9821166992188,
+      "stability/token_entropy_mean": 4.19140625,
+      "step": 100
+    },
+    {
+      "epoch": 0.5374280230326296,
+      "grad_norm": 0.11625107377767563,
+      "kl": 0.038600411266088486,
+      "learning_rate": 9.999815554351224e-07,
+      "logits/chosen": -59427038.072289154,
+      "logits/rejected": -53858540.051948056,
+      "logps/chosen": -399.85146837349396,
+      "logps/rejected": -209.96349939123377,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999867379665375,
+      "loss/total_with_kl": 0.5004057884216309,
+      "metrics/advantage_var": 0.0838162824511528,
+      "regularization/advantage_var": 0.0838162824511528,
+      "regularization/kl": 0.0004190813924651593,
+      "rewards/chosen": 0.0005104481186493333,
+      "rewards/margins": 0.00010723205834995542,
+      "rewards/rejected": 0.00040321606029937793,
+      "stability/repetition_rate_mean": 0.2053571343421936,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 29.89505386352539,
+      "stability/response_length_var": 893.7142944335938,
+      "stability/token_entropy_mean": 3.41796875,
+      "step": 105
+    },
+    {
+      "epoch": 0.5630198336532309,
+      "grad_norm": 0.09513776749372482,
+      "kl": 0.034523267298936844,
+      "learning_rate": 9.999066267225444e-07,
+      "logits/chosen": -57464671.61445783,
+      "logits/rejected": -53675948.883116886,
+      "logps/chosen": -378.51877823795184,
+      "logps/rejected": -206.05215097402598,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999869465827942,
+      "loss/total_with_kl": 0.5003308653831482,
+      "metrics/advantage_var": 0.06877034902572632,
+      "regularization/advantage_var": 0.06877034902572632,
+      "regularization/kl": 0.00034385171602480114,
+      "rewards/chosen": 0.0004449200827673257,
+      "rewards/margins": 8.504402561005693e-05,
+      "rewards/rejected": 0.0003598760571572688,
+      "stability/repetition_rate_mean": 0.321445494890213,
+      "stability/response_length_mean": 22.625,
+      "stability/response_length_std": 24.68624496459961,
+      "stability/response_length_var": 609.4107055664062,
+      "stability/token_entropy_mean": 4.3740234375,
+      "step": 110
+    },
+    {
+      "epoch": 0.5886116442738324,
+      "grad_norm": 0.08769083023071289,
+      "kl": 0.04661265388131142,
+      "learning_rate": 9.997740697079592e-07,
+      "logits/chosen": -60680289.215189874,
+      "logits/rejected": -53502097.38271605,
+      "logps/chosen": -364.5057357594937,
+      "logps/rejected": -201.1548514660494,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999448359012604,
+      "loss/total_with_kl": 0.5003185272216797,
+      "metrics/advantage_var": 0.07474813610315323,
+      "regularization/advantage_var": 0.07474813610315323,
+      "regularization/kl": 0.00037374068051576614,
+      "rewards/chosen": 0.0008680501693411718,
+      "rewards/margins": 0.000450884205235189,
+      "rewards/rejected": 0.0004171659641059828,
+      "stability/repetition_rate_mean": 0.3046875,
+      "stability/response_length_mean": 41.125,
+      "stability/response_length_std": 31.60215950012207,
+      "stability/response_length_var": 998.6964111328125,
+      "stability/token_entropy_mean": 4.880859375,
+      "step": 115
+    },
+    {
+      "epoch": 0.6142034548944337,
+      "grad_norm": 0.09087492525577545,
+      "kl": 0.02683141827583313,
+      "learning_rate": 9.995838996722914e-07,
+      "logits/chosen": -57850217.4117647,
+      "logits/rejected": -54498101.65269461,
+      "logps/chosen": -384.176087622549,
+      "logps/rejected": -212.20452844311376,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993181228637695,
+      "loss/total_with_kl": 0.5003178119659424,
+      "metrics/advantage_var": 0.07719477266073227,
+      "regularization/advantage_var": 0.07719477266073227,
+      "regularization/kl": 0.000385973893571645,
+      "rewards/chosen": 0.0006882845381506128,
+      "rewards/margins": 0.0005699411889836399,
+      "rewards/rejected": 0.00011834334916697291,
+      "stability/repetition_rate_mean": 0.18938209116458893,
+      "stability/response_length_mean": 29.375,
+      "stability/response_length_std": 29.548688888549805,
+      "stability/response_length_var": 873.125,
+      "stability/token_entropy_mean": 5.3125,
+      "step": 120
+    },
+    {
+      "epoch": 0.6397952655150352,
+      "grad_norm": 0.08841491490602493,
+      "kl": 0.03785186633467674,
+      "learning_rate": 9.993361385379874e-07,
+      "logits/chosen": -60808739.06849315,
+      "logits/rejected": -55804009.93103448,
+      "logps/chosen": -417.3874946489726,
+      "logps/rejected": -216.35508710488506,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000187158584595,
+      "loss/total_with_kl": 0.500385046005249,
+      "metrics/advantage_var": 0.0732690691947937,
+      "regularization/advantage_var": 0.0732690691947937,
+      "regularization/kl": 0.0003663453389890492,
+      "rewards/chosen": -8.181337751958468e-05,
+      "rewards/margins": -0.00020296169137557255,
+      "rewards/rejected": 0.00012114831385598786,
+      "stability/repetition_rate_mean": 0.15937499701976776,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 24.93276596069336,
+      "stability/response_length_var": 621.6428833007812,
+      "stability/token_entropy_mean": 3.63671875,
+      "step": 125
+    },
+    {
+      "epoch": 0.6653870761356366,
+      "grad_norm": 0.09630157798528671,
+      "kl": 0.044555749744176865,
+      "learning_rate": 9.99030814866488e-07,
+      "logits/chosen": -57510337.23870968,
+      "logits/rejected": -53975995.733333334,
+      "logps/chosen": -392.6136340725806,
+      "logps/rejected": -183.91416903409092,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49994951486587524,
+      "loss/total_with_kl": 0.5003226399421692,
+      "metrics/advantage_var": 0.0746237188577652,
+      "regularization/advantage_var": 0.0746237188577652,
+      "regularization/kl": 0.000373118557035923,
+      "rewards/chosen": 0.00044304853485476587,
+      "rewards/margins": 0.00039567723542539255,
+      "rewards/rejected": 4.737129942937331e-05,
+      "stability/repetition_rate_mean": 0.23289062082767487,
+      "stability/response_length_mean": 43.25,
+      "stability/response_length_std": 29.620214462280273,
+      "stability/response_length_var": 877.3571166992188,
+      "stability/token_entropy_mean": 3.642578125,
+      "step": 130
+    },
+    {
+      "epoch": 0.690978886756238,
+      "grad_norm": 0.10103689879179001,
+      "kl": 0.04840748757123947,
+      "learning_rate": 9.986679638549358e-07,
+      "logits/chosen": -59991988.61349693,
+      "logits/rejected": -55199457.01910828,
+      "logps/chosen": -401.72205713190186,
+      "logps/rejected": -247.8357882165605,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999224841594696,
+      "loss/total_with_kl": 0.5003660321235657,
+      "metrics/advantage_var": 0.08870525658130646,
+      "regularization/advantage_var": 0.08870525658130646,
+      "regularization/kl": 0.00044352628174237907,
+      "rewards/chosen": 0.0009412271844828787,
+      "rewards/margins": 0.000591533045224002,
+      "rewards/rejected": 0.00034969413925887673,
+      "stability/repetition_rate_mean": 0.2230902761220932,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 30.20406723022461,
+      "stability/response_length_var": 912.2857055664062,
+      "stability/token_entropy_mean": 3.40625,
+      "step": 135
+    },
+    {
+      "epoch": 0.7165706973768394,
+      "grad_norm": 0.12567856907844543,
+      "kl": 0.03631413355469704,
+      "learning_rate": 9.982476273321175e-07,
+      "logits/chosen": -57247570.23030303,
+      "logits/rejected": -50254762.11612903,
+      "logps/chosen": -367.1258049242424,
+      "logps/rejected": -190.0154485887097,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993759393692017,
+      "loss/total_with_kl": 0.5003074407577515,
+      "metrics/advantage_var": 0.0739670991897583,
+      "regularization/advantage_var": 0.0739670991897583,
+      "regularization/kl": 0.00036983549944125116,
+      "rewards/chosen": 0.000639494214997147,
+      "rewards/margins": 0.0004491325897526065,
+      "rewards/rejected": 0.00019036162524454055,
+      "stability/repetition_rate_mean": 0.359375,
+      "stability/response_length_mean": 56.25,
+      "stability/response_length_std": 21.920310974121094,
+      "stability/response_length_var": 480.5,
+      "stability/token_entropy_mean": 4.75,
+      "step": 140
+    },
+    {
+      "epoch": 0.7421625079974408,
+      "grad_norm": 0.09891662001609802,
+      "kl": 0.056184422224760056,
+      "learning_rate": 9.977698537536417e-07,
+      "logits/chosen": -51311412.43373494,
+      "logits/rejected": -51875494.233766235,
+      "logps/chosen": -330.2195500753012,
+      "logps/rejected": -199.49284699675326,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993544816970825,
+      "loss/total_with_kl": 0.5002543926239014,
+      "metrics/advantage_var": 0.06378551572561264,
+      "regularization/advantage_var": 0.06378551572561264,
+      "regularization/kl": 0.00031892754486761987,
+      "rewards/chosen": 0.0005227750144809125,
+      "rewards/margins": 0.0005721743595297488,
+      "rewards/rejected": -4.93993450488363e-05,
+      "stability/repetition_rate_mean": 0.24448327720165253,
+      "stability/response_length_mean": 28.875,
+      "stability/response_length_std": 29.73423957824707,
+      "stability/response_length_var": 884.125,
+      "stability/token_entropy_mean": 5.236328125,
+      "step": 145
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.10929251462221146,
+      "kl": 0.05055147036910057,
+      "learning_rate": 9.972346981963546e-07,
+      "logits/chosen": -55581629.74117647,
+      "logits/rejected": -50397125.97333334,
+      "logps/chosen": -397.8387867647059,
+      "logps/rejected": -207.7371484375,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998582899570465,
+      "loss/total_with_kl": 0.5003008842468262,
+      "metrics/advantage_var": 0.08850937336683273,
+      "regularization/advantage_var": 0.08850937336683273,
+      "regularization/kl": 0.00044254687963984907,
+      "rewards/chosen": 0.0009563709883128895,
+      "rewards/margins": 0.001068605093979368,
+      "rewards/rejected": -0.00011223410566647848,
+      "stability/repetition_rate_mean": 0.2115885466337204,
+      "stability/response_length_mean": 28.25,
+      "stability/response_length_std": 29.90341567993164,
+      "stability/response_length_var": 894.2142944335938,
+      "stability/token_entropy_mean": 4.80859375,
+      "step": 150
+    },
+    {
+      "epoch": 0.7933461292386437,
+      "grad_norm": 0.09307374805212021,
+      "kl": 0.048836734145879745,
+      "learning_rate": 9.966422223519885e-07,
+      "logits/chosen": -54227281.31764706,
+      "logits/rejected": -49584203.093333334,
+      "logps/chosen": -360.93809742647056,
+      "logps/rejected": -191.300625,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999396502971649,
+      "loss/total_with_kl": 0.5003570318222046,
+      "metrics/advantage_var": 0.08347625285387039,
+      "regularization/advantage_var": 0.08347625285387039,
+      "regularization/kl": 0.00041738126310519874,
+      "rewards/chosen": 0.0005118872312938465,
+      "rewards/margins": 0.00046203332277489637,
+      "rewards/rejected": 4.985390851895015e-05,
+      "stability/repetition_rate_mean": 0.3074128031730652,
+      "stability/response_length_mean": 61.375,
+      "stability/response_length_std": 7.424621105194092,
+      "stability/response_length_var": 55.125,
+      "stability/token_entropy_mean": 4.6044921875,
+      "step": 155
+    },
+    {
+      "epoch": 0.818937939859245,
+      "grad_norm": 0.09413865953683853,
+      "kl": 0.0488591343164444,
+      "learning_rate": 9.959924945200523e-07,
+      "logits/chosen": -55468525.71428572,
+      "logits/rejected": -49233495.578947365,
+      "logps/chosen": -416.8404947916667,
+      "logps/rejected": -186.8857293379934,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999455511569977,
+      "loss/total_with_kl": 0.5003138780593872,
+      "metrics/advantage_var": 0.07364356517791748,
+      "regularization/advantage_var": 0.07364356517791748,
+      "regularization/kl": 0.00036821779212914407,
+      "rewards/chosen": 0.0005480412039018813,
+      "rewards/margins": 0.0004319082198659878,
+      "rewards/rejected": 0.00011613298403589349,
+      "stability/repetition_rate_mean": 0.291015625,
+      "stability/response_length_mean": 26.25,
+      "stability/response_length_std": 31.535694122314453,
+      "stability/response_length_var": 994.5,
+      "stability/token_entropy_mean": 3.7998046875,
+      "step": 160
+    },
+    {
+      "epoch": 0.8445297504798465,
+      "grad_norm": 0.10981868207454681,
+      "kl": 0.04979108273983002,
+      "learning_rate": 9.952855895999568e-07,
+      "logits/chosen": -54661708.505747125,
+      "logits/rejected": -50090075.17808219,
+      "logps/chosen": -370.8274066091954,
+      "logps/rejected": -212.58053296232876,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999002516269684,
+      "loss/total_with_kl": 0.5002889633178711,
+      "metrics/advantage_var": 0.07773963361978531,
+      "regularization/advantage_var": 0.07773963361978531,
+      "regularization/kl": 0.0003886981576215476,
+      "rewards/chosen": 0.0008934761943488285,
+      "rewards/margins": 0.0007776661556483329,
+      "rewards/rejected": 0.00011581003870049568,
+      "stability/repetition_rate_mean": 0.2632460594177246,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 23.150131225585938,
+      "stability/response_length_var": 535.9285888671875,
+      "stability/token_entropy_mean": 4.81640625,
+      "step": 165
+    },
+    {
+      "epoch": 0.8701215611004478,
+      "grad_norm": 0.08388755470514297,
+      "kl": 0.06272337585687637,
+      "learning_rate": 9.9452158908238e-07,
+      "logits/chosen": -59387522.27218935,
+      "logits/rejected": -52949961.74834437,
+      "logps/chosen": -441.2528661242604,
+      "logps/rejected": -191.2500776076159,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49989959597587585,
+      "loss/total_with_kl": 0.5002941489219666,
+      "metrics/advantage_var": 0.07889693975448608,
+      "regularization/advantage_var": 0.07889693975448608,
+      "regularization/kl": 0.0003944846976082772,
+      "rewards/chosen": 0.000889729113268429,
+      "rewards/margins": 0.0008121537112671523,
+      "rewards/rejected": 7.757540200127671e-05,
+      "stability/repetition_rate_mean": 0.2809279263019562,
+      "stability/response_length_mean": 28.25,
+      "stability/response_length_std": 30.844078063964844,
+      "stability/response_length_var": 951.3571166992188,
+      "stability/token_entropy_mean": 3.09033203125,
+      "step": 170
+    },
+    {
+      "epoch": 0.8957133717210493,
+      "grad_norm": 0.09992633014917374,
+      "kl": 0.06644636392593384,
+      "learning_rate": 9.937005810398745e-07,
+      "logits/chosen": -56929020.915662654,
+      "logits/rejected": -48413912.103896104,
+      "logps/chosen": -356.35577466114455,
+      "logps/rejected": -194.76746398133116,
+      "loss": 0.5003,
+      "loss/base_kto": 0.499949187040329,
+      "loss/total_with_kl": 0.5003606677055359,
+      "metrics/advantage_var": 0.08229832351207733,
+      "regularization/advantage_var": 0.08229832351207733,
+      "regularization/kl": 0.0004114916082471609,
+      "rewards/chosen": 0.0008141423564359367,
+      "rewards/margins": 0.0004135140424331689,
+      "rewards/rejected": 0.0004006283140027678,
+      "stability/repetition_rate_mean": 0.28187286853790283,
+      "stability/response_length_mean": 32.5,
+      "stability/response_length_std": 27.969371795654297,
+      "stability/response_length_var": 782.2857055664062,
+      "stability/token_entropy_mean": 4.7412109375,
+      "step": 175
+    },
+    {
+      "epoch": 0.9213051823416507,
+      "grad_norm": 0.09833401441574097,
+      "kl": 0.0622967854142189,
+      "learning_rate": 9.928226601167138e-07,
+      "logits/chosen": -52938997.00613497,
+      "logits/rejected": -48348688.30573248,
+      "logps/chosen": -376.0574194785276,
+      "logps/rejected": -211.23741042993632,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999646246433258,
+      "loss/total_with_kl": 0.500411868095398,
+      "metrics/advantage_var": 0.08944838494062424,
+      "regularization/advantage_var": 0.08944838494062424,
+      "regularization/kl": 0.00044724190956912935,
+      "rewards/chosen": 0.0008839887161196375,
+      "rewards/margins": 0.00027612949146912696,
+      "rewards/rejected": 0.0006078592246505105,
+      "stability/repetition_rate_mean": 0.16015625,
+      "stability/response_length_mean": 40.375,
+      "stability/response_length_std": 32.61874008178711,
+      "stability/response_length_var": 1063.982177734375,
+      "stability/token_entropy_mean": 3.53125,
+      "step": 180
+    },
+    {
+      "epoch": 0.946896992962252,
+      "grad_norm": 0.09617482125759125,
+      "kl": 0.0709846243262291,
+      "learning_rate": 9.918879275179817e-07,
+      "logits/chosen": -54943105.58024691,
+      "logits/rejected": -49885858.025316454,
+      "logps/chosen": -311.92582947530866,
+      "logps/rejected": -197.01087816455697,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999426007270813,
+      "loss/total_with_kl": 0.5003011226654053,
+      "metrics/advantage_var": 0.07169583439826965,
+      "regularization/advantage_var": 0.07169583439826965,
+      "regularization/kl": 0.0003584791556932032,
+      "rewards/chosen": 0.0007554439850795417,
+      "rewards/margins": 0.0004727014776067187,
+      "rewards/rejected": 0.00028274250747282293,
+      "stability/repetition_rate_mean": 0.18998579680919647,
+      "stability/response_length_mean": 22.25,
+      "stability/response_length_std": 27.78874397277832,
+      "stability/response_length_var": 772.2142944335938,
+      "stability/token_entropy_mean": 3.828125,
+      "step": 185
+    },
+    {
+      "epoch": 0.9724888035828535,
+      "grad_norm": 0.10346374660730362,
+      "kl": 0.05659087374806404,
+      "learning_rate": 9.90896490997906e-07,
+      "logits/chosen": -53424790.22754491,
+      "logits/rejected": -50103520.20915033,
+      "logps/chosen": -392.2973194236527,
+      "logps/rejected": -211.16874744689542,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998948276042938,
+      "loss/total_with_kl": 0.5002766847610474,
+      "metrics/advantage_var": 0.07637263834476471,
+      "regularization/advantage_var": 0.07637263834476471,
+      "regularization/kl": 0.0003818631812464446,
+      "rewards/chosen": 0.0010801159336181459,
+      "rewards/margins": 0.000781740584588929,
+      "rewards/rejected": 0.0002983753490292169,
+      "stability/repetition_rate_mean": 0.1047247052192688,
+      "stability/response_length_mean": 11.375,
+      "stability/response_length_std": 21.738296508789062,
+      "stability/response_length_var": 472.5535583496094,
+      "stability/token_entropy_mean": 5.38671875,
+      "step": 190
+    },
+    {
+      "epoch": 0.9980806142034548,
+      "grad_norm": 0.10313382744789124,
+      "kl": 0.055899590253829956,
+      "learning_rate": 9.898484648474362e-07,
+      "logits/chosen": -60530562.477419354,
+      "logits/rejected": -54311222.303030305,
+      "logps/chosen": -392.6547379032258,
+      "logps/rejected": -197.9125946969697,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998457431793213,
+      "loss/total_with_kl": 0.5002177357673645,
+      "metrics/advantage_var": 0.07440383732318878,
+      "regularization/advantage_var": 0.07440383732318878,
+      "regularization/kl": 0.00037201918894425035,
+      "rewards/chosen": 0.0013020292405159244,
+      "rewards/margins": 0.0012345370724578758,
+      "rewards/rejected": 6.74921680580486e-05,
+      "stability/repetition_rate_mean": 0.3515625,
+      "stability/response_length_mean": 40.875,
+      "stability/response_length_std": 31.96175193786621,
+      "stability/response_length_var": 1021.5535888671875,
+      "stability/token_entropy_mean": 4.490234375,
+      "step": 195
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 196,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.1861866257511875e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

mistral7b_kto_kl-beta0.01/checkpoint-196/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85beda43fa0c145c1439cc242873864caf2fddd3411f37e00a57d3806e34bafa
+size 8017

mistral7b_kto_kl-beta0.01/checkpoint-196/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

mistral7b_kto_kl-beta0.01/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mistral7b_kto_kl-beta0.01/stability_metrics.jsonl

@@ -0,0 +1,38 @@
+{"stability/response_length_mean": 46.125, "stability/response_length_std": 27.231481552124023, "stability/response_length_var": 741.5535888671875, "stability/token_entropy_mean": 3.33642578125, "stability/repetition_rate_mean": 0.3590347170829773, "step": 10}
+{"stability/response_length_mean": 29.375, "stability/response_length_std": 29.92341423034668, "stability/response_length_var": 895.4107055664062, "stability/token_entropy_mean": 3.7041015625, "stability/repetition_rate_mean": 0.2929687201976776, "step": 15}
+{"stability/response_length_mean": 37.25, "stability/response_length_std": 29.040611267089844, "stability/response_length_var": 843.3571166992188, "stability/token_entropy_mean": 4.49609375, "stability/repetition_rate_mean": 0.20810391008853912, "step": 20}
+{"stability/response_length_mean": 50.25, "stability/response_length_std": 25.549671173095703, "stability/response_length_var": 652.7857055664062, "stability/token_entropy_mean": 4.5546875, "stability/repetition_rate_mean": 0.18344351649284363, "step": 25}
+{"stability/response_length_mean": 43.0, "stability/response_length_std": 29.602123260498047, "stability/response_length_var": 876.2857055664062, "stability/token_entropy_mean": 4.66015625, "stability/repetition_rate_mean": 0.1583891361951828, "step": 30}
+{"stability/response_length_mean": 53.375, "stability/response_length_std": 21.14533042907715, "stability/response_length_var": 447.125, "stability/token_entropy_mean": 5.330078125, "stability/repetition_rate_mean": 0.33708456158638, "step": 35}
+{"stability/response_length_mean": 35.0, "stability/response_length_std": 31.518701553344727, "stability/response_length_var": 993.4285888671875, "stability/token_entropy_mean": 4.89453125, "stability/repetition_rate_mean": 0.16694079339504242, "step": 40}
+{"stability/response_length_mean": 28.875, "stability/response_length_std": 29.87324333190918, "stability/response_length_var": 892.4107055664062, "stability/token_entropy_mean": 4.8203125, "stability/repetition_rate_mean": 0.0974486693739891, "step": 45}
+{"stability/response_length_mean": 54.625, "stability/response_length_std": 16.044023513793945, "stability/response_length_var": 257.41070556640625, "stability/token_entropy_mean": 3.8291015625, "stability/repetition_rate_mean": 0.3931290805339813, "step": 50}
+{"stability/response_length_mean": 18.625, "stability/response_length_std": 28.560400009155273, "stability/response_length_var": 815.6964111328125, "stability/token_entropy_mean": 4.59375, "stability/repetition_rate_mean": 0.07375919073820114, "step": 55}
+{"stability/response_length_mean": 51.125, "stability/response_length_std": 21.90523338317871, "stability/response_length_var": 479.83929443359375, "stability/token_entropy_mean": 5.30078125, "stability/repetition_rate_mean": 0.21421611309051514, "step": 60}
+{"stability/response_length_mean": 20.625, "stability/response_length_std": 27.176342010498047, "stability/response_length_var": 738.5535888671875, "stability/token_entropy_mean": 5.548828125, "stability/repetition_rate_mean": 0.1857219785451889, "step": 65}
+{"stability/response_length_mean": 33.875, "stability/response_length_std": 32.36151885986328, "stability/response_length_var": 1047.267822265625, "stability/token_entropy_mean": 5.71875, "stability/repetition_rate_mean": 0.185546875, "step": 70}
+{"stability/response_length_mean": 22.75, "stability/response_length_std": 31.07479476928711, "stability/response_length_var": 965.6428833007812, "stability/token_entropy_mean": 2.1484375, "stability/repetition_rate_mean": 0.08548677712678909, "step": 75}
+{"stability/response_length_mean": 40.25, "stability/response_length_std": 29.421810150146484, "stability/response_length_var": 865.6428833007812, "stability/token_entropy_mean": 5.23828125, "stability/repetition_rate_mean": 0.18572917580604553, "step": 80}
+{"stability/response_length_mean": 28.0, "stability/response_length_std": 31.181495666503906, "stability/response_length_var": 972.2857055664062, "stability/token_entropy_mean": 4.95703125, "stability/repetition_rate_mean": 0.1350446343421936, "step": 85}
+{"stability/response_length_mean": 26.625, "stability/response_length_std": 27.79998779296875, "stability/response_length_var": 772.8392944335938, "stability/token_entropy_mean": 4.29296875, "stability/repetition_rate_mean": 0.2426609843969345, "step": 90}
+{"stability/response_length_mean": 27.0, "stability/response_length_std": 31.1264705657959, "stability/response_length_var": 968.8571166992188, "stability/token_entropy_mean": 3.6875, "stability/repetition_rate_mean": 0.20546108484268188, "step": 95}
+{"stability/response_length_mean": 37.875, "stability/response_length_std": 30.248672485351562, "stability/response_length_var": 914.9821166992188, "stability/token_entropy_mean": 4.19140625, "stability/repetition_rate_mean": 0.23228970170021057, "step": 100}
+{"stability/response_length_mean": 29.0, "stability/response_length_std": 29.89505386352539, "stability/response_length_var": 893.7142944335938, "stability/token_entropy_mean": 3.41796875, "stability/repetition_rate_mean": 0.2053571343421936, "step": 105}
+{"stability/response_length_mean": 22.625, "stability/response_length_std": 24.68624496459961, "stability/response_length_var": 609.4107055664062, "stability/token_entropy_mean": 4.3740234375, "stability/repetition_rate_mean": 0.321445494890213, "step": 110}
+{"stability/response_length_mean": 41.125, "stability/response_length_std": 31.60215950012207, "stability/response_length_var": 998.6964111328125, "stability/token_entropy_mean": 4.880859375, "stability/repetition_rate_mean": 0.3046875, "step": 115}
+{"stability/response_length_mean": 29.375, "stability/response_length_std": 29.548688888549805, "stability/response_length_var": 873.125, "stability/token_entropy_mean": 5.3125, "stability/repetition_rate_mean": 0.18938209116458893, "step": 120}
+{"stability/response_length_mean": 15.25, "stability/response_length_std": 24.93276596069336, "stability/response_length_var": 621.6428833007812, "stability/token_entropy_mean": 3.63671875, "stability/repetition_rate_mean": 0.15937499701976776, "step": 125}
+{"stability/response_length_mean": 43.25, "stability/response_length_std": 29.620214462280273, "stability/response_length_var": 877.3571166992188, "stability/token_entropy_mean": 3.642578125, "stability/repetition_rate_mean": 0.23289062082767487, "step": 130}
+{"stability/response_length_mean": 29.0, "stability/response_length_std": 30.20406723022461, "stability/response_length_var": 912.2857055664062, "stability/token_entropy_mean": 3.40625, "stability/repetition_rate_mean": 0.2230902761220932, "step": 135}
+{"stability/response_length_mean": 56.25, "stability/response_length_std": 21.920310974121094, "stability/response_length_var": 480.5, "stability/token_entropy_mean": 4.75, "stability/repetition_rate_mean": 0.359375, "step": 140}
+{"stability/response_length_mean": 28.875, "stability/response_length_std": 29.73423957824707, "stability/response_length_var": 884.125, "stability/token_entropy_mean": 5.236328125, "stability/repetition_rate_mean": 0.24448327720165253, "step": 145}
+{"stability/response_length_mean": 28.25, "stability/response_length_std": 29.90341567993164, "stability/response_length_var": 894.2142944335938, "stability/token_entropy_mean": 4.80859375, "stability/repetition_rate_mean": 0.2115885466337204, "step": 150}
+{"stability/response_length_mean": 61.375, "stability/response_length_std": 7.424621105194092, "stability/response_length_var": 55.125, "stability/token_entropy_mean": 4.6044921875, "stability/repetition_rate_mean": 0.3074128031730652, "step": 155}
+{"stability/response_length_mean": 26.25, "stability/response_length_std": 31.535694122314453, "stability/response_length_var": 994.5, "stability/token_entropy_mean": 3.7998046875, "stability/repetition_rate_mean": 0.291015625, "step": 160}
+{"stability/response_length_mean": 15.25, "stability/response_length_std": 23.150131225585938, "stability/response_length_var": 535.9285888671875, "stability/token_entropy_mean": 4.81640625, "stability/repetition_rate_mean": 0.2632460594177246, "step": 165}
+{"stability/response_length_mean": 28.25, "stability/response_length_std": 30.844078063964844, "stability/response_length_var": 951.3571166992188, "stability/token_entropy_mean": 3.09033203125, "stability/repetition_rate_mean": 0.2809279263019562, "step": 170}
+{"stability/response_length_mean": 32.5, "stability/response_length_std": 27.969371795654297, "stability/response_length_var": 782.2857055664062, "stability/token_entropy_mean": 4.7412109375, "stability/repetition_rate_mean": 0.28187286853790283, "step": 175}
+{"stability/response_length_mean": 40.375, "stability/response_length_std": 32.61874008178711, "stability/response_length_var": 1063.982177734375, "stability/token_entropy_mean": 3.53125, "stability/repetition_rate_mean": 0.16015625, "step": 180}
+{"stability/response_length_mean": 22.25, "stability/response_length_std": 27.78874397277832, "stability/response_length_var": 772.2142944335938, "stability/token_entropy_mean": 3.828125, "stability/repetition_rate_mean": 0.18998579680919647, "step": 185}
+{"stability/response_length_mean": 11.375, "stability/response_length_std": 21.738296508789062, "stability/response_length_var": 472.5535583496094, "stability/token_entropy_mean": 5.38671875, "stability/repetition_rate_mean": 0.1047247052192688, "step": 190}
+{"stability/response_length_mean": 40.875, "stability/response_length_std": 31.96175193786621, "stability/response_length_var": 1021.5535888671875, "stability/token_entropy_mean": 4.490234375, "stability/repetition_rate_mean": 0.3515625, "step": 195}

mistral7b_kto_kl-beta0.01/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

mistral7b_kto_kl-beta0.01/tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

mistral7b_kto_kl-beta0.01/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.1861866257511875e+18,
+    "train_loss": 0.5003062401499067,
+    "train_runtime": 3152.1109,
+    "train_samples_per_second": 31.725,
+    "train_steps_per_second": 0.062
+}

mistral7b_kto_kl-beta0.01/trainer_log.jsonl

@@ -0,0 +1,40 @@
+{"current_steps": 5, "total_steps": 196, "loss": 0.5002, "lr": 4e-08, "epoch": 0.025591810620601407, "percentage": 2.55, "elapsed_time": "0:01:18", "remaining_time": "0:49:40", "rewards/chosen": 0.00018095341511070728, "logps/chosen": -439.02749794407896, "logits/chosen": -59719296.0, "rewards/rejected": -1.9088354227798326e-05, "logps/rejected": -198.41131882440476, "logits/rejected": -50418438.0952381, "rewards/margins": 0.0002000417693385056, "regularization/kl": 0.00019087796681560576, "regularization/advantage_var": 0.03817559406161308, "kl": 0.00977869238704443}
+{"current_steps": 10, "total_steps": 196, "loss": 0.5003, "lr": 9e-08, "epoch": 0.05118362124120281, "percentage": 5.1, "elapsed_time": "0:02:38", "remaining_time": "0:49:13", "rewards/chosen": 0.00024344235078080907, "logps/chosen": -429.914163961039, "logits/chosen": -59294786.49350649, "rewards/rejected": 0.0002721029457198568, "logps/rejected": -192.84071442018072, "logits/rejected": -50011718.93975904, "rewards/margins": -2.866059493904772e-05, "regularization/kl": 0.00034622737439349294, "regularization/advantage_var": 0.06924548000097275, "kl": 0.0236990787088871, "stability/response_length_mean": 46.125, "stability/response_length_std": 27.231481552124023, "stability/response_length_var": 741.5535888671875, "stability/token_entropy_mean": 3.33642578125, "stability/repetition_rate_mean": 0.3590347170829773}
+{"current_steps": 15, "total_steps": 196, "loss": 0.5003, "lr": 1.4e-07, "epoch": 0.07677543186180422, "percentage": 7.65, "elapsed_time": "0:03:58", "remaining_time": "0:47:58", "rewards/chosen": 8.117372565197222e-05, "logps/chosen": -404.1227272727273, "logits/chosen": -56242337.35757576, "rewards/rejected": 0.00013670010432120292, "logps/rejected": -200.6803931451613, "logits/rejected": -51928440.56774194, "rewards/margins": -5.552637866923071e-05, "regularization/kl": 0.000302851025480777, "regularization/advantage_var": 0.06057020649313927, "kl": 0.014778117649257183, "stability/response_length_mean": 29.375, "stability/response_length_std": 29.92341423034668, "stability/response_length_var": 895.4107055664062, "stability/token_entropy_mean": 3.7041015625, "stability/repetition_rate_mean": 0.2929687201976776}
+{"current_steps": 20, "total_steps": 196, "loss": 0.5004, "lr": 1.8999999999999998e-07, "epoch": 0.10236724248240563, "percentage": 10.2, "elapsed_time": "0:05:20", "remaining_time": "0:46:56", "rewards/chosen": -5.6106073316186666e-05, "logps/chosen": -343.7023193359375, "logits/chosen": -51660729.6, "rewards/rejected": 0.00027608638629317284, "logps/rejected": -225.6357421875, "logits/rejected": -48058240.0, "rewards/margins": -0.0003321924596093595, "regularization/kl": 0.00032614529482088983, "regularization/advantage_var": 0.06522905826568604, "kl": 0.0199257992208004, "stability/response_length_mean": 37.25, "stability/response_length_std": 29.040611267089844, "stability/response_length_var": 843.3571166992188, "stability/token_entropy_mean": 4.49609375, "stability/repetition_rate_mean": 0.20810391008853912}
+{"current_steps": 25, "total_steps": 196, "loss": 0.5003, "lr": 2.4e-07, "epoch": 0.12795905310300704, "percentage": 12.76, "elapsed_time": "0:06:39", "remaining_time": "0:45:35", "rewards/chosen": 3.336018015598429e-05, "logps/chosen": -378.48811961206894, "logits/chosen": -56484991.11724138, "rewards/rejected": -8.565639810902731e-06, "logps/rejected": -202.79370535714287, "logits/rejected": -52026502.58285714, "rewards/margins": 4.192581996688702e-05, "regularization/kl": 0.0003036776324734092, "regularization/advantage_var": 0.06073552370071411, "kl": 0.01614060439169407, "stability/response_length_mean": 50.25, "stability/response_length_std": 25.549671173095703, "stability/response_length_var": 652.7857055664062, "stability/token_entropy_mean": 4.5546875, "stability/repetition_rate_mean": 0.18344351649284363}
+{"current_steps": 30, "total_steps": 196, "loss": 0.5003, "lr": 2.9e-07, "epoch": 0.15355086372360843, "percentage": 15.31, "elapsed_time": "0:07:59", "remaining_time": "0:44:15", "rewards/chosen": 0.00012335383752048415, "logps/chosen": -468.56894924496646, "logits/chosen": -57725168.536912754, "rewards/rejected": 0.00010796357490863019, "logps/rejected": -198.23707054093566, "logits/rejected": -51507990.456140354, "rewards/margins": 1.539026261185396e-05, "regularization/kl": 0.0003661524970084429, "regularization/advantage_var": 0.07323050498962402, "kl": 0.016464022919535637, "stability/response_length_mean": 43.0, "stability/response_length_std": 29.602123260498047, "stability/response_length_var": 876.2857055664062, "stability/token_entropy_mean": 4.66015625, "stability/repetition_rate_mean": 0.1583891361951828}
+{"current_steps": 35, "total_steps": 196, "loss": 0.5004, "lr": 3.4000000000000003e-07, "epoch": 0.17914267434420986, "percentage": 17.86, "elapsed_time": "0:09:21", "remaining_time": "0:43:01", "rewards/chosen": -9.798232607449158e-06, "logps/chosen": -385.9375, "logits/chosen": -57637447.29113924, "rewards/rejected": 6.529041508465638e-05, "logps/rejected": -195.11993634259258, "logits/rejected": -53896343.7037037, "rewards/margins": -7.508864769210554e-05, "regularization/kl": 0.0002908978785853833, "regularization/advantage_var": 0.058179568499326706, "kl": 0.019741442054510117, "stability/response_length_mean": 53.375, "stability/response_length_std": 21.14533042907715, "stability/response_length_var": 447.125, "stability/token_entropy_mean": 5.330078125, "stability/repetition_rate_mean": 0.33708456158638}
+{"current_steps": 40, "total_steps": 196, "loss": 0.5004, "lr": 3.8999999999999997e-07, "epoch": 0.20473448496481125, "percentage": 20.41, "elapsed_time": "0:10:41", "remaining_time": "0:41:43", "rewards/chosen": 7.171009201556444e-05, "logps/chosen": -421.8955078125, "logits/chosen": -56632742.4, "rewards/rejected": 0.0004529234021902084, "logps/rejected": -219.26396484375, "logits/rejected": -50238966.4, "rewards/margins": -0.000381213310174644, "regularization/kl": 0.00038220261922106147, "regularization/advantage_var": 0.07644052803516388, "kl": 0.021908169612288475, "stability/response_length_mean": 35.0, "stability/response_length_std": 31.518701553344727, "stability/response_length_var": 993.4285888671875, "stability/token_entropy_mean": 4.89453125, "stability/repetition_rate_mean": 0.16694079339504242}
+{"current_steps": 45, "total_steps": 196, "loss": 0.5003, "lr": 4.3999999999999997e-07, "epoch": 0.23032629558541268, "percentage": 22.96, "elapsed_time": "0:12:02", "remaining_time": "0:40:24", "rewards/chosen": 0.00010591877886912967, "logps/chosen": -352.3178743708054, "logits/chosen": -52694314.95302013, "rewards/rejected": 0.0001813181899270119, "logps/rejected": -243.85603983918128, "logits/rejected": -50773680.65497076, "rewards/margins": -7.539941105788222e-05, "regularization/kl": 0.000331446120981127, "regularization/advantage_var": 0.06628922373056412, "kl": 0.034686531871557236, "stability/response_length_mean": 28.875, "stability/response_length_std": 29.87324333190918, "stability/response_length_var": 892.4107055664062, "stability/token_entropy_mean": 4.8203125, "stability/repetition_rate_mean": 0.0974486693739891}
+{"current_steps": 50, "total_steps": 196, "loss": 0.5003, "lr": 4.9e-07, "epoch": 0.2559181062060141, "percentage": 25.51, "elapsed_time": "0:13:22", "remaining_time": "0:39:03", "rewards/chosen": 0.0003667769816097797, "logps/chosen": -375.1624370805369, "logits/chosen": -54048891.70469799, "rewards/rejected": 0.00020532222867709156, "logps/rejected": -180.2200749269006, "logits/rejected": -48298175.62573099, "rewards/margins": 0.00016145475293268815, "regularization/kl": 0.00027261764626018703, "regularization/advantage_var": 0.0545235350728035, "kl": 0.036549247801303864, "stability/response_length_mean": 54.625, "stability/response_length_std": 16.044023513793945, "stability/response_length_var": 257.41070556640625, "stability/token_entropy_mean": 3.8291015625, "stability/repetition_rate_mean": 0.3931290805339813}
+{"current_steps": 55, "total_steps": 196, "loss": 0.5003, "lr": 5.4e-07, "epoch": 0.28150991682661547, "percentage": 28.06, "elapsed_time": "0:14:42", "remaining_time": "0:37:41", "rewards/chosen": 0.0002302746780170417, "logps/chosen": -344.4889848602484, "logits/chosen": -56953518.9068323, "rewards/rejected": 0.0002661707266321722, "logps/rejected": -230.42143278301887, "logits/rejected": -49572674.012578614, "rewards/margins": -3.589604861513053e-05, "regularization/kl": 0.0002471174520906061, "regularization/advantage_var": 0.04942348971962929, "kl": 0.03960302472114563, "stability/response_length_mean": 18.625, "stability/response_length_std": 28.560400009155273, "stability/response_length_var": 815.6964111328125, "stability/token_entropy_mean": 4.59375, "stability/repetition_rate_mean": 0.07375919073820114}
+{"current_steps": 60, "total_steps": 196, "loss": 0.5003, "lr": 5.9e-07, "epoch": 0.30710172744721687, "percentage": 30.61, "elapsed_time": "0:16:03", "remaining_time": "0:36:24", "rewards/chosen": 0.0003821046966494936, "logps/chosen": -359.6825994318182, "logits/chosen": -52676881.06666667, "rewards/rejected": 0.0002780948675447895, "logps/rejected": -178.19910534274194, "logits/rejected": -47836245.88387097, "rewards/margins": 0.00010400982910470412, "regularization/kl": 0.0002799059438984841, "regularization/advantage_var": 0.05598118528723717, "kl": 0.03357061371207237, "stability/response_length_mean": 51.125, "stability/response_length_std": 21.90523338317871, "stability/response_length_var": 479.83929443359375, "stability/token_entropy_mean": 5.30078125, "stability/repetition_rate_mean": 0.21421611309051514}
+{"current_steps": 65, "total_steps": 196, "loss": 0.5003, "lr": 6.4e-07, "epoch": 0.3326935380678183, "percentage": 33.16, "elapsed_time": "0:17:24", "remaining_time": "0:35:04", "rewards/chosen": 0.0003270598428864633, "logps/chosen": -381.9140877016129, "logits/chosen": -57574505.70322581, "rewards/rejected": 0.00041834359819238837, "logps/rejected": -183.5405303030303, "logits/rejected": -47286880.193939395, "rewards/margins": -9.128375530592506e-05, "regularization/kl": 0.0003493055410217494, "regularization/advantage_var": 0.06986110657453537, "kl": 0.045144613832235336, "stability/response_length_mean": 20.625, "stability/response_length_std": 27.176342010498047, "stability/response_length_var": 738.5535888671875, "stability/token_entropy_mean": 5.548828125, "stability/repetition_rate_mean": 0.1857219785451889}
+{"current_steps": 70, "total_steps": 196, "loss": 0.5003, "lr": 6.9e-07, "epoch": 0.3582853486884197, "percentage": 35.71, "elapsed_time": "0:18:44", "remaining_time": "0:33:43", "rewards/chosen": 0.00039446188076849907, "logps/chosen": -379.0890120967742, "logits/chosen": -52786750.76129032, "rewards/rejected": 0.00014293173497373409, "logps/rejected": -220.40785984848486, "logits/rejected": -49049150.06060606, "rewards/margins": 0.00025153014579476496, "regularization/kl": 0.00034651882015168667, "regularization/advantage_var": 0.06930377334356308, "kl": 0.04333776980638504, "stability/response_length_mean": 33.875, "stability/response_length_std": 32.36151885986328, "stability/response_length_var": 1047.267822265625, "stability/token_entropy_mean": 5.71875, "stability/repetition_rate_mean": 0.185546875}
+{"current_steps": 75, "total_steps": 196, "loss": 0.5003, "lr": 7.4e-07, "epoch": 0.3838771593090211, "percentage": 38.27, "elapsed_time": "0:20:05", "remaining_time": "0:32:24", "rewards/chosen": 0.000722566871224223, "logps/chosen": -314.19855891047297, "logits/chosen": -49513603.45945946, "rewards/rejected": 0.00028177757942399314, "logps/rejected": -228.85667242005815, "logits/rejected": -47910197.58139535, "rewards/margins": 0.00044078929180022984, "regularization/kl": 0.00032720837043598294, "regularization/advantage_var": 0.06544167548418045, "kl": 0.03558317944407463, "stability/response_length_mean": 22.75, "stability/response_length_std": 31.07479476928711, "stability/response_length_var": 965.6428833007812, "stability/token_entropy_mean": 2.1484375, "stability/repetition_rate_mean": 0.08548677712678909}
+{"current_steps": 80, "total_steps": 196, "loss": 0.5003, "lr": 7.9e-07, "epoch": 0.4094689699296225, "percentage": 40.82, "elapsed_time": "0:21:26", "remaining_time": "0:31:05", "rewards/chosen": 0.0004708408717288087, "logps/chosen": -403.9830298013245, "logits/chosen": -58642201.430463575, "rewards/rejected": 0.00010233398901640311, "logps/rejected": -201.04558062130178, "logits/rejected": -53484331.92899408, "rewards/margins": 0.00036850688271240556, "regularization/kl": 0.0002758416812866926, "regularization/advantage_var": 0.05516833811998367, "kl": 0.04469752684235573, "stability/response_length_mean": 40.25, "stability/response_length_std": 29.421810150146484, "stability/response_length_var": 865.6428833007812, "stability/token_entropy_mean": 5.23828125, "stability/repetition_rate_mean": 0.18572917580604553}
+{"current_steps": 85, "total_steps": 196, "loss": 0.5003, "lr": 8.399999999999999e-07, "epoch": 0.4350607805502239, "percentage": 43.37, "elapsed_time": "0:22:46", "remaining_time": "0:29:44", "rewards/chosen": 0.00048093819810498146, "logps/chosen": -358.2844254032258, "logits/chosen": -51504801.85806452, "rewards/rejected": 0.00020758288376259082, "logps/rejected": -230.40402462121213, "logits/rejected": -47172201.5030303, "rewards/margins": 0.00027335531434239063, "regularization/kl": 0.00030950302607379854, "regularization/advantage_var": 0.061900604516267776, "kl": 0.038033999502658844, "stability/response_length_mean": 28.0, "stability/response_length_std": 31.181495666503906, "stability/response_length_var": 972.2857055664062, "stability/token_entropy_mean": 4.95703125, "stability/repetition_rate_mean": 0.1350446343421936}
+{"current_steps": 90, "total_steps": 196, "loss": 0.5003, "lr": 8.9e-07, "epoch": 0.46065259117082535, "percentage": 45.92, "elapsed_time": "0:24:07", "remaining_time": "0:28:25", "rewards/chosen": 0.00039930832691681693, "logps/chosen": -380.7494491185897, "logits/chosen": -58452775.384615384, "rewards/rejected": 0.0002680534678624897, "logps/rejected": -187.19999285442074, "logits/rejected": -50602352.3902439, "rewards/margins": 0.0001312548590543272, "regularization/kl": 0.0003422955924179405, "regularization/advantage_var": 0.06845911592245102, "kl": 0.04362564906477928, "stability/response_length_mean": 26.625, "stability/response_length_std": 27.79998779296875, "stability/response_length_var": 772.8392944335938, "stability/token_entropy_mean": 4.29296875, "stability/repetition_rate_mean": 0.2426609843969345}
+{"current_steps": 95, "total_steps": 196, "loss": 0.5003, "lr": 9.399999999999999e-07, "epoch": 0.48624440179142675, "percentage": 48.47, "elapsed_time": "0:25:27", "remaining_time": "0:27:04", "rewards/chosen": 0.00026683563578732404, "logps/chosen": -314.9666732594937, "logits/chosen": -50427842.43037975, "rewards/rejected": 9.55940764627339e-05, "logps/rejected": -207.13416280864197, "logits/rejected": -49013747.358024694, "rewards/margins": 0.00017124155932459015, "regularization/kl": 0.00030967846396379173, "regularization/advantage_var": 0.06193568930029869, "kl": 0.030433058738708496, "stability/response_length_mean": 27.0, "stability/response_length_std": 31.1264705657959, "stability/response_length_var": 968.8571166992188, "stability/token_entropy_mean": 3.6875, "stability/repetition_rate_mean": 0.20546108484268188}
+{"current_steps": 100, "total_steps": 196, "loss": 0.5003, "lr": 9.9e-07, "epoch": 0.5118362124120281, "percentage": 51.02, "elapsed_time": "0:26:48", "remaining_time": "0:25:43", "rewards/chosen": 0.0008716205688747199, "logps/chosen": -359.78301836993245, "logits/chosen": -54771206.91891892, "rewards/rejected": 4.605338692145292e-05, "logps/rejected": -216.57428688226744, "logits/rejected": -50969665.488372095, "rewards/margins": 0.000825567181953267, "regularization/kl": 0.00034040678292512894, "regularization/advantage_var": 0.06808135658502579, "kl": 0.03825628012418747, "stability/response_length_mean": 37.875, "stability/response_length_std": 30.248672485351562, "stability/response_length_var": 914.9821166992188, "stability/token_entropy_mean": 4.19140625, "stability/repetition_rate_mean": 0.23228970170021057}
+{"current_steps": 105, "total_steps": 196, "loss": 0.5003, "lr": 9.999815554351224e-07, "epoch": 0.5374280230326296, "percentage": 53.57, "elapsed_time": "0:28:09", "remaining_time": "0:24:23", "rewards/chosen": 0.0005104481186493333, "logps/chosen": -399.85146837349396, "logits/chosen": -59427038.072289154, "rewards/rejected": 0.00040321606029937793, "logps/rejected": -209.96349939123377, "logits/rejected": -53858540.051948056, "rewards/margins": 0.00010723205834995542, "regularization/kl": 0.0004190813924651593, "regularization/advantage_var": 0.0838162824511528, "kl": 0.038600411266088486, "stability/response_length_mean": 29.0, "stability/response_length_std": 29.89505386352539, "stability/response_length_var": 893.7142944335938, "stability/token_entropy_mean": 3.41796875, "stability/repetition_rate_mean": 0.2053571343421936}
+{"current_steps": 110, "total_steps": 196, "loss": 0.5003, "lr": 9.999066267225444e-07, "epoch": 0.5630198336532309, "percentage": 56.12, "elapsed_time": "0:29:30", "remaining_time": "0:23:04", "rewards/chosen": 0.0004449200827673257, "logps/chosen": -378.51877823795184, "logits/chosen": -57464671.61445783, "rewards/rejected": 0.0003598760571572688, "logps/rejected": -206.05215097402598, "logits/rejected": -53675948.883116886, "rewards/margins": 8.504402561005693e-05, "regularization/kl": 0.00034385171602480114, "regularization/advantage_var": 0.06877034902572632, "kl": 0.034523267298936844, "stability/response_length_mean": 22.625, "stability/response_length_std": 24.68624496459961, "stability/response_length_var": 609.4107055664062, "stability/token_entropy_mean": 4.3740234375, "stability/repetition_rate_mean": 0.321445494890213}
+{"current_steps": 115, "total_steps": 196, "loss": 0.5003, "lr": 9.997740697079592e-07, "epoch": 0.5886116442738324, "percentage": 58.67, "elapsed_time": "0:30:51", "remaining_time": "0:21:44", "rewards/chosen": 0.0008680501693411718, "logps/chosen": -364.5057357594937, "logits/chosen": -60680289.215189874, "rewards/rejected": 0.0004171659641059828, "logps/rejected": -201.1548514660494, "logits/rejected": -53502097.38271605, "rewards/margins": 0.000450884205235189, "regularization/kl": 0.00037374068051576614, "regularization/advantage_var": 0.07474813610315323, "kl": 0.04661265388131142, "stability/response_length_mean": 41.125, "stability/response_length_std": 31.60215950012207, "stability/response_length_var": 998.6964111328125, "stability/token_entropy_mean": 4.880859375, "stability/repetition_rate_mean": 0.3046875}
+{"current_steps": 120, "total_steps": 196, "loss": 0.5003, "lr": 9.995838996722914e-07, "epoch": 0.6142034548944337, "percentage": 61.22, "elapsed_time": "0:32:11", "remaining_time": "0:20:23", "rewards/chosen": 0.0006882845381506128, "logps/chosen": -384.176087622549, "logits/chosen": -57850217.4117647, "rewards/rejected": 0.00011834334916697291, "logps/rejected": -212.20452844311376, "logits/rejected": -54498101.65269461, "rewards/margins": 0.0005699411889836399, "regularization/kl": 0.000385973893571645, "regularization/advantage_var": 0.07719477266073227, "kl": 0.02683141827583313, "stability/response_length_mean": 29.375, "stability/response_length_std": 29.548688888549805, "stability/response_length_var": 873.125, "stability/token_entropy_mean": 5.3125, "stability/repetition_rate_mean": 0.18938209116458893}
+{"current_steps": 125, "total_steps": 196, "loss": 0.5003, "lr": 9.993361385379874e-07, "epoch": 0.6397952655150352, "percentage": 63.78, "elapsed_time": "0:33:31", "remaining_time": "0:19:02", "rewards/chosen": -8.181337751958468e-05, "logps/chosen": -417.3874946489726, "logits/chosen": -60808739.06849315, "rewards/rejected": 0.00012114831385598786, "logps/rejected": -216.35508710488506, "logits/rejected": -55804009.93103448, "rewards/margins": -0.00020296169137557255, "regularization/kl": 0.0003663453389890492, "regularization/advantage_var": 0.0732690691947937, "kl": 0.03785186633467674, "stability/response_length_mean": 15.25, "stability/response_length_std": 24.93276596069336, "stability/response_length_var": 621.6428833007812, "stability/token_entropy_mean": 3.63671875, "stability/repetition_rate_mean": 0.15937499701976776}
+{"current_steps": 130, "total_steps": 196, "loss": 0.5003, "lr": 9.99030814866488e-07, "epoch": 0.6653870761356366, "percentage": 66.33, "elapsed_time": "0:34:52", "remaining_time": "0:17:42", "rewards/chosen": 0.00044304853485476587, "logps/chosen": -392.6136340725806, "logits/chosen": -57510337.23870968, "rewards/rejected": 4.737129942937331e-05, "logps/rejected": -183.91416903409092, "logits/rejected": -53975995.733333334, "rewards/margins": 0.00039567723542539255, "regularization/kl": 0.000373118557035923, "regularization/advantage_var": 0.0746237188577652, "kl": 0.044555749744176865, "stability/response_length_mean": 43.25, "stability/response_length_std": 29.620214462280273, "stability/response_length_var": 877.3571166992188, "stability/token_entropy_mean": 3.642578125, "stability/repetition_rate_mean": 0.23289062082767487}
+{"current_steps": 135, "total_steps": 196, "loss": 0.5003, "lr": 9.986679638549358e-07, "epoch": 0.690978886756238, "percentage": 68.88, "elapsed_time": "0:36:12", "remaining_time": "0:16:21", "rewards/chosen": 0.0009412271844828787, "logps/chosen": -401.72205713190186, "logits/chosen": -59991988.61349693, "rewards/rejected": 0.00034969413925887673, "logps/rejected": -247.8357882165605, "logits/rejected": -55199457.01910828, "rewards/margins": 0.000591533045224002, "regularization/kl": 0.00044352628174237907, "regularization/advantage_var": 0.08870525658130646, "kl": 0.04840748757123947, "stability/response_length_mean": 29.0, "stability/response_length_std": 30.20406723022461, "stability/response_length_var": 912.2857055664062, "stability/token_entropy_mean": 3.40625, "stability/repetition_rate_mean": 0.2230902761220932}
+{"current_steps": 140, "total_steps": 196, "loss": 0.5003, "lr": 9.982476273321175e-07, "epoch": 0.7165706973768394, "percentage": 71.43, "elapsed_time": "0:37:32", "remaining_time": "0:15:01", "rewards/chosen": 0.000639494214997147, "logps/chosen": -367.1258049242424, "logits/chosen": -57247570.23030303, "rewards/rejected": 0.00019036162524454055, "logps/rejected": -190.0154485887097, "logits/rejected": -50254762.11612903, "rewards/margins": 0.0004491325897526065, "regularization/kl": 0.00036983549944125116, "regularization/advantage_var": 0.0739670991897583, "kl": 0.03631413355469704, "stability/response_length_mean": 56.25, "stability/response_length_std": 21.920310974121094, "stability/response_length_var": 480.5, "stability/token_entropy_mean": 4.75, "stability/repetition_rate_mean": 0.359375}
+{"current_steps": 145, "total_steps": 196, "loss": 0.5003, "lr": 9.977698537536417e-07, "epoch": 0.7421625079974408, "percentage": 73.98, "elapsed_time": "0:38:52", "remaining_time": "0:13:40", "rewards/chosen": 0.0005227750144809125, "logps/chosen": -330.2195500753012, "logits/chosen": -51311412.43373494, "rewards/rejected": -4.93993450488363e-05, "logps/rejected": -199.49284699675326, "logits/rejected": -51875494.233766235, "rewards/margins": 0.0005721743595297488, "regularization/kl": 0.00031892754486761987, "regularization/advantage_var": 0.06378551572561264, "kl": 0.056184422224760056, "stability/response_length_mean": 28.875, "stability/response_length_std": 29.73423957824707, "stability/response_length_var": 884.125, "stability/token_entropy_mean": 5.236328125, "stability/repetition_rate_mean": 0.24448327720165253}
+{"current_steps": 150, "total_steps": 196, "loss": 0.5003, "lr": 9.972346981963546e-07, "epoch": 0.7677543186180422, "percentage": 76.53, "elapsed_time": "0:40:13", "remaining_time": "0:12:20", "rewards/chosen": 0.0009563709883128895, "logps/chosen": -397.8387867647059, "logits/chosen": -55581629.74117647, "rewards/rejected": -0.00011223410566647848, "logps/rejected": -207.7371484375, "logits/rejected": -50397125.97333334, "rewards/margins": 0.001068605093979368, "regularization/kl": 0.00044254687963984907, "regularization/advantage_var": 0.08850937336683273, "kl": 0.05055147036910057, "stability/response_length_mean": 28.25, "stability/response_length_std": 29.90341567993164, "stability/response_length_var": 894.2142944335938, "stability/token_entropy_mean": 4.80859375, "stability/repetition_rate_mean": 0.2115885466337204}
+{"current_steps": 155, "total_steps": 196, "loss": 0.5003, "lr": 9.966422223519885e-07, "epoch": 0.7933461292386437, "percentage": 79.08, "elapsed_time": "0:41:35", "remaining_time": "0:11:00", "rewards/chosen": 0.0005118872312938465, "logps/chosen": -360.93809742647056, "logits/chosen": -54227281.31764706, "rewards/rejected": 4.985390851895015e-05, "logps/rejected": -191.300625, "logits/rejected": -49584203.093333334, "rewards/margins": 0.00046203332277489637, "regularization/kl": 0.00041738126310519874, "regularization/advantage_var": 0.08347625285387039, "kl": 0.048836734145879745, "stability/response_length_mean": 61.375, "stability/response_length_std": 7.424621105194092, "stability/response_length_var": 55.125, "stability/token_entropy_mean": 4.6044921875, "stability/repetition_rate_mean": 0.3074128031730652}
+{"current_steps": 160, "total_steps": 196, "loss": 0.5003, "lr": 9.959924945200523e-07, "epoch": 0.818937939859245, "percentage": 81.63, "elapsed_time": "0:42:55", "remaining_time": "0:09:39", "rewards/chosen": 0.0005480412039018813, "logps/chosen": -416.8404947916667, "logits/chosen": -55468525.71428572, "rewards/rejected": 0.00011613298403589349, "logps/rejected": -186.8857293379934, "logits/rejected": -49233495.578947365, "rewards/margins": 0.0004319082198659878, "regularization/kl": 0.00036821779212914407, "regularization/advantage_var": 0.07364356517791748, "kl": 0.0488591343164444, "stability/response_length_mean": 26.25, "stability/response_length_std": 31.535694122314453, "stability/response_length_var": 994.5, "stability/token_entropy_mean": 3.7998046875, "stability/repetition_rate_mean": 0.291015625}
+{"current_steps": 165, "total_steps": 196, "loss": 0.5003, "lr": 9.952855895999568e-07, "epoch": 0.8445297504798465, "percentage": 84.18, "elapsed_time": "0:44:15", "remaining_time": "0:08:18", "rewards/chosen": 0.0008934761943488285, "logps/chosen": -370.8274066091954, "logits/chosen": -54661708.505747125, "rewards/rejected": 0.00011581003870049568, "logps/rejected": -212.58053296232876, "logits/rejected": -50090075.17808219, "rewards/margins": 0.0007776661556483329, "regularization/kl": 0.0003886981576215476, "regularization/advantage_var": 0.07773963361978531, "kl": 0.04979108273983002, "stability/response_length_mean": 15.25, "stability/response_length_std": 23.150131225585938, "stability/response_length_var": 535.9285888671875, "stability/token_entropy_mean": 4.81640625, "stability/repetition_rate_mean": 0.2632460594177246}
+{"current_steps": 170, "total_steps": 196, "loss": 0.5003, "lr": 9.9452158908238e-07, "epoch": 0.8701215611004478, "percentage": 86.73, "elapsed_time": "0:45:37", "remaining_time": "0:06:58", "rewards/chosen": 0.000889729113268429, "logps/chosen": -441.2528661242604, "logits/chosen": -59387522.27218935, "rewards/rejected": 7.757540200127671e-05, "logps/rejected": -191.2500776076159, "logits/rejected": -52949961.74834437, "rewards/margins": 0.0008121537112671523, "regularization/kl": 0.0003944846976082772, "regularization/advantage_var": 0.07889693975448608, "kl": 0.06272337585687637, "stability/response_length_mean": 28.25, "stability/response_length_std": 30.844078063964844, "stability/response_length_var": 951.3571166992188, "stability/token_entropy_mean": 3.09033203125, "stability/repetition_rate_mean": 0.2809279263019562}
+{"current_steps": 175, "total_steps": 196, "loss": 0.5003, "lr": 9.937005810398745e-07, "epoch": 0.8957133717210493, "percentage": 89.29, "elapsed_time": "0:46:57", "remaining_time": "0:05:38", "rewards/chosen": 0.0008141423564359367, "logps/chosen": -356.35577466114455, "logits/chosen": -56929020.915662654, "rewards/rejected": 0.0004006283140027678, "logps/rejected": -194.76746398133116, "logits/rejected": -48413912.103896104, "rewards/margins": 0.0004135140424331689, "regularization/kl": 0.0004114916082471609, "regularization/advantage_var": 0.08229832351207733, "kl": 0.06644636392593384, "stability/response_length_mean": 32.5, "stability/response_length_std": 27.969371795654297, "stability/response_length_var": 782.2857055664062, "stability/token_entropy_mean": 4.7412109375, "stability/repetition_rate_mean": 0.28187286853790283}
+{"current_steps": 180, "total_steps": 196, "loss": 0.5003, "lr": 9.928226601167138e-07, "epoch": 0.9213051823416507, "percentage": 91.84, "elapsed_time": "0:48:18", "remaining_time": "0:04:17", "rewards/chosen": 0.0008839887161196375, "logps/chosen": -376.0574194785276, "logits/chosen": -52938997.00613497, "rewards/rejected": 0.0006078592246505105, "logps/rejected": -211.23741042993632, "logits/rejected": -48348688.30573248, "rewards/margins": 0.00027612949146912696, "regularization/kl": 0.00044724190956912935, "regularization/advantage_var": 0.08944838494062424, "kl": 0.0622967854142189, "stability/response_length_mean": 40.375, "stability/response_length_std": 32.61874008178711, "stability/response_length_var": 1063.982177734375, "stability/token_entropy_mean": 3.53125, "stability/repetition_rate_mean": 0.16015625}
+{"current_steps": 185, "total_steps": 196, "loss": 0.5003, "lr": 9.918879275179817e-07, "epoch": 0.946896992962252, "percentage": 94.39, "elapsed_time": "0:49:37", "remaining_time": "0:02:57", "rewards/chosen": 0.0007554439850795417, "logps/chosen": -311.92582947530866, "logits/chosen": -54943105.58024691, "rewards/rejected": 0.00028274250747282293, "logps/rejected": -197.01087816455697, "logits/rejected": -49885858.025316454, "rewards/margins": 0.0004727014776067187, "regularization/kl": 0.0003584791556932032, "regularization/advantage_var": 0.07169583439826965, "kl": 0.0709846243262291, "stability/response_length_mean": 22.25, "stability/response_length_std": 27.78874397277832, "stability/response_length_var": 772.2142944335938, "stability/token_entropy_mean": 3.828125, "stability/repetition_rate_mean": 0.18998579680919647}
+{"current_steps": 190, "total_steps": 196, "loss": 0.5003, "lr": 9.90896490997906e-07, "epoch": 0.9724888035828535, "percentage": 96.94, "elapsed_time": "0:50:57", "remaining_time": "0:01:36", "rewards/chosen": 0.0010801159336181459, "logps/chosen": -392.2973194236527, "logits/chosen": -53424790.22754491, "rewards/rejected": 0.0002983753490292169, "logps/rejected": -211.16874744689542, "logits/rejected": -50103520.20915033, "rewards/margins": 0.000781740584588929, "regularization/kl": 0.0003818631812464446, "regularization/advantage_var": 0.07637263834476471, "kl": 0.05659087374806404, "stability/response_length_mean": 11.375, "stability/response_length_std": 21.738296508789062, "stability/response_length_var": 472.5535583496094, "stability/token_entropy_mean": 5.38671875, "stability/repetition_rate_mean": 0.1047247052192688}
+{"current_steps": 195, "total_steps": 196, "loss": 0.5003, "lr": 9.898484648474362e-07, "epoch": 0.9980806142034548, "percentage": 99.49, "elapsed_time": "0:52:18", "remaining_time": "0:00:16", "rewards/chosen": 0.0013020292405159244, "logps/chosen": -392.6547379032258, "logits/chosen": -60530562.477419354, "rewards/rejected": 6.74921680580486e-05, "logps/rejected": -197.9125946969697, "logits/rejected": -54311222.303030305, "rewards/margins": 0.0012345370724578758, "regularization/kl": 0.00037201918894425035, "regularization/advantage_var": 0.07440383732318878, "kl": 0.055899590253829956, "stability/response_length_mean": 40.875, "stability/response_length_std": 31.96175193786621, "stability/response_length_var": 1021.5535888671875, "stability/token_entropy_mean": 4.490234375, "stability/repetition_rate_mean": 0.3515625}
+{"current_steps": 196, "total_steps": 196, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:52:32", "remaining_time": "0:00:00"}

mistral7b_kto_kl-beta0.01/trainer_state.json

@@ -0,0 +1,1013 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 196,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.025591810620601407,
+      "grad_norm": 0.10970883816480637,
+      "kl": 0.00977869238704443,
+      "learning_rate": 4e-08,
+      "logits/chosen": -59719296.0,
+      "logits/rejected": -50418438.0952381,
+      "logps/chosen": -439.02749794407896,
+      "logps/rejected": -198.41131882440476,
+      "loss": 0.5002,
+      "loss/base_kto": 0.49997520446777344,
+      "loss/total_with_kl": 0.5001660585403442,
+      "metrics/advantage_var": 0.03817559406161308,
+      "regularization/advantage_var": 0.03817559406161308,
+      "regularization/kl": 0.00019087796681560576,
+      "rewards/chosen": 0.00018095341511070728,
+      "rewards/margins": 0.0002000417693385056,
+      "rewards/rejected": -1.9088354227798326e-05,
+      "step": 5
+    },
+    {
+      "epoch": 0.05118362124120281,
+      "grad_norm": 0.09480192512273788,
+      "kl": 0.0236990787088871,
+      "learning_rate": 9e-08,
+      "logits/chosen": -59294786.49350649,
+      "logits/rejected": -50011718.93975904,
+      "logps/chosen": -429.914163961039,
+      "logps/rejected": -192.84071442018072,
+      "loss": 0.5003,
+      "loss/base_kto": 0.500006377696991,
+      "loss/total_with_kl": 0.5003525614738464,
+      "metrics/advantage_var": 0.06924548000097275,
+      "regularization/advantage_var": 0.06924548000097275,
+      "regularization/kl": 0.00034622737439349294,
+      "rewards/chosen": 0.00024344235078080907,
+      "rewards/margins": -2.866059493904772e-05,
+      "rewards/rejected": 0.0002721029457198568,
+      "stability/repetition_rate_mean": 0.3590347170829773,
+      "stability/response_length_mean": 46.125,
+      "stability/response_length_std": 27.231481552124023,
+      "stability/response_length_var": 741.5535888671875,
+      "stability/token_entropy_mean": 3.33642578125,
+      "step": 10
+    },
+    {
+      "epoch": 0.07677543186180422,
+      "grad_norm": 0.11248399317264557,
+      "kl": 0.014778117649257183,
+      "learning_rate": 1.4e-07,
+      "logits/chosen": -56242337.35757576,
+      "logits/rejected": -51928440.56774194,
+      "logps/chosen": -404.1227272727273,
+      "logps/rejected": -200.6803931451613,
+      "loss": 0.5003,
+      "loss/base_kto": 0.500006914138794,
+      "loss/total_with_kl": 0.5003097653388977,
+      "metrics/advantage_var": 0.06057020649313927,
+      "regularization/advantage_var": 0.06057020649313927,
+      "regularization/kl": 0.000302851025480777,
+      "rewards/chosen": 8.117372565197222e-05,
+      "rewards/margins": -5.552637866923071e-05,
+      "rewards/rejected": 0.00013670010432120292,
+      "stability/repetition_rate_mean": 0.2929687201976776,
+      "stability/response_length_mean": 29.375,
+      "stability/response_length_std": 29.92341423034668,
+      "stability/response_length_var": 895.4107055664062,
+      "stability/token_entropy_mean": 3.7041015625,
+      "step": 15
+    },
+    {
+      "epoch": 0.10236724248240563,
+      "grad_norm": 0.11460169404745102,
+      "kl": 0.0199257992208004,
+      "learning_rate": 1.8999999999999998e-07,
+      "logits/chosen": -51660729.6,
+      "logits/rejected": -48058240.0,
+      "logps/chosen": -343.7023193359375,
+      "logps/rejected": -225.6357421875,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000439286231995,
+      "loss/total_with_kl": 0.5003700852394104,
+      "metrics/advantage_var": 0.06522905826568604,
+      "regularization/advantage_var": 0.06522905826568604,
+      "regularization/kl": 0.00032614529482088983,
+      "rewards/chosen": -5.6106073316186666e-05,
+      "rewards/margins": -0.0003321924596093595,
+      "rewards/rejected": 0.00027608638629317284,
+      "stability/repetition_rate_mean": 0.20810391008853912,
+      "stability/response_length_mean": 37.25,
+      "stability/response_length_std": 29.040611267089844,
+      "stability/response_length_var": 843.3571166992188,
+      "stability/token_entropy_mean": 4.49609375,
+      "step": 20
+    },
+    {
+      "epoch": 0.12795905310300704,
+      "grad_norm": 0.11199703812599182,
+      "kl": 0.01614060439169407,
+      "learning_rate": 2.4e-07,
+      "logits/chosen": -56484991.11724138,
+      "logits/rejected": -52026502.58285714,
+      "logps/chosen": -378.48811961206894,
+      "logps/rejected": -202.79370535714287,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999929368495941,
+      "loss/total_with_kl": 0.5002965927124023,
+      "metrics/advantage_var": 0.06073552370071411,
+      "regularization/advantage_var": 0.06073552370071411,
+      "regularization/kl": 0.0003036776324734092,
+      "rewards/chosen": 3.336018015598429e-05,
+      "rewards/margins": 4.192581996688702e-05,
+      "rewards/rejected": -8.565639810902731e-06,
+      "stability/repetition_rate_mean": 0.18344351649284363,
+      "stability/response_length_mean": 50.25,
+      "stability/response_length_std": 25.549671173095703,
+      "stability/response_length_var": 652.7857055664062,
+      "stability/token_entropy_mean": 4.5546875,
+      "step": 25
+    },
+    {
+      "epoch": 0.15355086372360843,
+      "grad_norm": 0.10699352622032166,
+      "kl": 0.016464022919535637,
+      "learning_rate": 2.9e-07,
+      "logits/chosen": -57725168.536912754,
+      "logits/rejected": -51507990.456140354,
+      "logps/chosen": -468.56894924496646,
+      "logps/rejected": -198.23707054093566,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999980032444,
+      "loss/total_with_kl": 0.5003641247749329,
+      "metrics/advantage_var": 0.07323050498962402,
+      "regularization/advantage_var": 0.07323050498962402,
+      "regularization/kl": 0.0003661524970084429,
+      "rewards/chosen": 0.00012335383752048415,
+      "rewards/margins": 1.539026261185396e-05,
+      "rewards/rejected": 0.00010796357490863019,
+      "stability/repetition_rate_mean": 0.1583891361951828,
+      "stability/response_length_mean": 43.0,
+      "stability/response_length_std": 29.602123260498047,
+      "stability/response_length_var": 876.2857055664062,
+      "stability/token_entropy_mean": 4.66015625,
+      "step": 30
+    },
+    {
+      "epoch": 0.17914267434420986,
+      "grad_norm": 0.21138133108615875,
+      "kl": 0.019741442054510117,
+      "learning_rate": 3.4000000000000003e-07,
+      "logits/chosen": -57637447.29113924,
+      "logits/rejected": -53896343.7037037,
+      "logps/chosen": -385.9375,
+      "logps/rejected": -195.11993634259258,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000119805335999,
+      "loss/total_with_kl": 0.5003029108047485,
+      "metrics/advantage_var": 0.058179568499326706,
+      "regularization/advantage_var": 0.058179568499326706,
+      "regularization/kl": 0.0002908978785853833,
+      "rewards/chosen": -9.798232607449158e-06,
+      "rewards/margins": -7.508864769210554e-05,
+      "rewards/rejected": 6.529041508465638e-05,
+      "stability/repetition_rate_mean": 0.33708456158638,
+      "stability/response_length_mean": 53.375,
+      "stability/response_length_std": 21.14533042907715,
+      "stability/response_length_var": 447.125,
+      "stability/token_entropy_mean": 5.330078125,
+      "step": 35
+    },
+    {
+      "epoch": 0.20473448496481125,
+      "grad_norm": 0.1126021295785904,
+      "kl": 0.021908169612288475,
+      "learning_rate": 3.8999999999999997e-07,
+      "logits/chosen": -56632742.4,
+      "logits/rejected": -50238966.4,
+      "logps/chosen": -421.8955078125,
+      "logps/rejected": -219.26396484375,
+      "loss": 0.5004,
+      "loss/base_kto": 0.5000482797622681,
+      "loss/total_with_kl": 0.5004304647445679,
+      "metrics/advantage_var": 0.07644052803516388,
+      "regularization/advantage_var": 0.07644052803516388,
+      "regularization/kl": 0.00038220261922106147,
+      "rewards/chosen": 7.171009201556444e-05,
+      "rewards/margins": -0.000381213310174644,
+      "rewards/rejected": 0.0004529234021902084,
+      "stability/repetition_rate_mean": 0.16694079339504242,
+      "stability/response_length_mean": 35.0,
+      "stability/response_length_std": 31.518701553344727,
+      "stability/response_length_var": 993.4285888671875,
+      "stability/token_entropy_mean": 4.89453125,
+      "step": 40
+    },
+    {
+      "epoch": 0.23032629558541268,
+      "grad_norm": 0.11807987093925476,
+      "kl": 0.034686531871557236,
+      "learning_rate": 4.3999999999999997e-07,
+      "logits/chosen": -52694314.95302013,
+      "logits/rejected": -50773680.65497076,
+      "logps/chosen": -352.3178743708054,
+      "logps/rejected": -243.85603983918128,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49999910593032837,
+      "loss/total_with_kl": 0.5003305673599243,
+      "metrics/advantage_var": 0.06628922373056412,
+      "regularization/advantage_var": 0.06628922373056412,
+      "regularization/kl": 0.000331446120981127,
+      "rewards/chosen": 0.00010591877886912967,
+      "rewards/margins": -7.539941105788222e-05,
+      "rewards/rejected": 0.0001813181899270119,
+      "stability/repetition_rate_mean": 0.0974486693739891,
+      "stability/response_length_mean": 28.875,
+      "stability/response_length_std": 29.87324333190918,
+      "stability/response_length_var": 892.4107055664062,
+      "stability/token_entropy_mean": 4.8203125,
+      "step": 45
+    },
+    {
+      "epoch": 0.2559181062060141,
+      "grad_norm": 0.08920294046401978,
+      "kl": 0.036549247801303864,
+      "learning_rate": 4.9e-07,
+      "logits/chosen": -54048891.70469799,
+      "logits/rejected": -48298175.62573099,
+      "logps/chosen": -375.1624370805369,
+      "logps/rejected": -180.2200749269006,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999710023403168,
+      "loss/total_with_kl": 0.500243604183197,
+      "metrics/advantage_var": 0.0545235350728035,
+      "regularization/advantage_var": 0.0545235350728035,
+      "regularization/kl": 0.00027261764626018703,
+      "rewards/chosen": 0.0003667769816097797,
+      "rewards/margins": 0.00016145475293268815,
+      "rewards/rejected": 0.00020532222867709156,
+      "stability/repetition_rate_mean": 0.3931290805339813,
+      "stability/response_length_mean": 54.625,
+      "stability/response_length_std": 16.044023513793945,
+      "stability/response_length_var": 257.41070556640625,
+      "stability/token_entropy_mean": 3.8291015625,
+      "step": 50
+    },
+    {
+      "epoch": 0.28150991682661547,
+      "grad_norm": 0.12241534143686295,
+      "kl": 0.03960302472114563,
+      "learning_rate": 5.4e-07,
+      "logits/chosen": -56953518.9068323,
+      "logits/rejected": -49572674.012578614,
+      "logps/chosen": -344.4889848602484,
+      "logps/rejected": -230.42143278301887,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000047087669373,
+      "loss/total_with_kl": 0.500251829624176,
+      "metrics/advantage_var": 0.04942348971962929,
+      "regularization/advantage_var": 0.04942348971962929,
+      "regularization/kl": 0.0002471174520906061,
+      "rewards/chosen": 0.0002302746780170417,
+      "rewards/margins": -3.589604861513053e-05,
+      "rewards/rejected": 0.0002661707266321722,
+      "stability/repetition_rate_mean": 0.07375919073820114,
+      "stability/response_length_mean": 18.625,
+      "stability/response_length_std": 28.560400009155273,
+      "stability/response_length_var": 815.6964111328125,
+      "stability/token_entropy_mean": 4.59375,
+      "step": 55
+    },
+    {
+      "epoch": 0.30710172744721687,
+      "grad_norm": 0.09351801872253418,
+      "kl": 0.03357061371207237,
+      "learning_rate": 5.9e-07,
+      "logits/chosen": -52676881.06666667,
+      "logits/rejected": -47836245.88387097,
+      "logps/chosen": -359.6825994318182,
+      "logps/rejected": -178.19910534274194,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999830424785614,
+      "loss/total_with_kl": 0.5002629160881042,
+      "metrics/advantage_var": 0.05598118528723717,
+      "regularization/advantage_var": 0.05598118528723717,
+      "regularization/kl": 0.0002799059438984841,
+      "rewards/chosen": 0.0003821046966494936,
+      "rewards/margins": 0.00010400982910470412,
+      "rewards/rejected": 0.0002780948675447895,
+      "stability/repetition_rate_mean": 0.21421611309051514,
+      "stability/response_length_mean": 51.125,
+      "stability/response_length_std": 21.90523338317871,
+      "stability/response_length_var": 479.83929443359375,
+      "stability/token_entropy_mean": 5.30078125,
+      "step": 60
+    },
+    {
+      "epoch": 0.3326935380678183,
+      "grad_norm": 0.10334884375333786,
+      "kl": 0.045144613832235336,
+      "learning_rate": 6.4e-07,
+      "logits/chosen": -57574505.70322581,
+      "logits/rejected": -47286880.193939395,
+      "logps/chosen": -381.9140877016129,
+      "logps/rejected": -183.5405303030303,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000074505805969,
+      "loss/total_with_kl": 0.5003567934036255,
+      "metrics/advantage_var": 0.06986110657453537,
+      "regularization/advantage_var": 0.06986110657453537,
+      "regularization/kl": 0.0003493055410217494,
+      "rewards/chosen": 0.0003270598428864633,
+      "rewards/margins": -9.128375530592506e-05,
+      "rewards/rejected": 0.00041834359819238837,
+      "stability/repetition_rate_mean": 0.1857219785451889,
+      "stability/response_length_mean": 20.625,
+      "stability/response_length_std": 27.176342010498047,
+      "stability/response_length_var": 738.5535888671875,
+      "stability/token_entropy_mean": 5.548828125,
+      "step": 65
+    },
+    {
+      "epoch": 0.3582853486884197,
+      "grad_norm": 0.08497199416160583,
+      "kl": 0.04333776980638504,
+      "learning_rate": 6.9e-07,
+      "logits/chosen": -52786750.76129032,
+      "logits/rejected": -49049150.06060606,
+      "logps/chosen": -379.0890120967742,
+      "logps/rejected": -220.40785984848486,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49996206164360046,
+      "loss/total_with_kl": 0.5003085732460022,
+      "metrics/advantage_var": 0.06930377334356308,
+      "regularization/advantage_var": 0.06930377334356308,
+      "regularization/kl": 0.00034651882015168667,
+      "rewards/chosen": 0.00039446188076849907,
+      "rewards/margins": 0.00025153014579476496,
+      "rewards/rejected": 0.00014293173497373409,
+      "stability/repetition_rate_mean": 0.185546875,
+      "stability/response_length_mean": 33.875,
+      "stability/response_length_std": 32.36151885986328,
+      "stability/response_length_var": 1047.267822265625,
+      "stability/token_entropy_mean": 5.71875,
+      "step": 70
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.10053345561027527,
+      "kl": 0.03558317944407463,
+      "learning_rate": 7.4e-07,
+      "logits/chosen": -49513603.45945946,
+      "logits/rejected": -47910197.58139535,
+      "logps/chosen": -314.19855891047297,
+      "logps/rejected": -228.85667242005815,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999457895755768,
+      "loss/total_with_kl": 0.5002729892730713,
+      "metrics/advantage_var": 0.06544167548418045,
+      "regularization/advantage_var": 0.06544167548418045,
+      "regularization/kl": 0.00032720837043598294,
+      "rewards/chosen": 0.000722566871224223,
+      "rewards/margins": 0.00044078929180022984,
+      "rewards/rejected": 0.00028177757942399314,
+      "stability/repetition_rate_mean": 0.08548677712678909,
+      "stability/response_length_mean": 22.75,
+      "stability/response_length_std": 31.07479476928711,
+      "stability/response_length_var": 965.6428833007812,
+      "stability/token_entropy_mean": 2.1484375,
+      "step": 75
+    },
+    {
+      "epoch": 0.4094689699296225,
+      "grad_norm": 0.09504130482673645,
+      "kl": 0.04469752684235573,
+      "learning_rate": 7.9e-07,
+      "logits/chosen": -58642201.430463575,
+      "logits/rejected": -53484331.92899408,
+      "logps/chosen": -403.9830298013245,
+      "logps/rejected": -201.04558062130178,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49995794892311096,
+      "loss/total_with_kl": 0.5002338290214539,
+      "metrics/advantage_var": 0.05516833811998367,
+      "regularization/advantage_var": 0.05516833811998367,
+      "regularization/kl": 0.0002758416812866926,
+      "rewards/chosen": 0.0004708408717288087,
+      "rewards/margins": 0.00036850688271240556,
+      "rewards/rejected": 0.00010233398901640311,
+      "stability/repetition_rate_mean": 0.18572917580604553,
+      "stability/response_length_mean": 40.25,
+      "stability/response_length_std": 29.421810150146484,
+      "stability/response_length_var": 865.6428833007812,
+      "stability/token_entropy_mean": 5.23828125,
+      "step": 80
+    },
+    {
+      "epoch": 0.4350607805502239,
+      "grad_norm": 0.10743357986211777,
+      "kl": 0.038033999502658844,
+      "learning_rate": 8.399999999999999e-07,
+      "logits/chosen": -51504801.85806452,
+      "logits/rejected": -47172201.5030303,
+      "logps/chosen": -358.2844254032258,
+      "logps/rejected": -230.40402462121213,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999658167362213,
+      "loss/total_with_kl": 0.5002753138542175,
+      "metrics/advantage_var": 0.061900604516267776,
+      "regularization/advantage_var": 0.061900604516267776,
+      "regularization/kl": 0.00030950302607379854,
+      "rewards/chosen": 0.00048093819810498146,
+      "rewards/margins": 0.00027335531434239063,
+      "rewards/rejected": 0.00020758288376259082,
+      "stability/repetition_rate_mean": 0.1350446343421936,
+      "stability/response_length_mean": 28.0,
+      "stability/response_length_std": 31.181495666503906,
+      "stability/response_length_var": 972.2857055664062,
+      "stability/token_entropy_mean": 4.95703125,
+      "step": 85
+    },
+    {
+      "epoch": 0.46065259117082535,
+      "grad_norm": 0.10124877840280533,
+      "kl": 0.04362564906477928,
+      "learning_rate": 8.9e-07,
+      "logits/chosen": -58452775.384615384,
+      "logits/rejected": -50602352.3902439,
+      "logps/chosen": -380.7494491185897,
+      "logps/rejected": -187.19999285442074,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49998241662979126,
+      "loss/total_with_kl": 0.5003246665000916,
+      "metrics/advantage_var": 0.06845911592245102,
+      "regularization/advantage_var": 0.06845911592245102,
+      "regularization/kl": 0.0003422955924179405,
+      "rewards/chosen": 0.00039930832691681693,
+      "rewards/margins": 0.0001312548590543272,
+      "rewards/rejected": 0.0002680534678624897,
+      "stability/repetition_rate_mean": 0.2426609843969345,
+      "stability/response_length_mean": 26.625,
+      "stability/response_length_std": 27.79998779296875,
+      "stability/response_length_var": 772.8392944335938,
+      "stability/token_entropy_mean": 4.29296875,
+      "step": 90
+    },
+    {
+      "epoch": 0.48624440179142675,
+      "grad_norm": 0.10090715438127518,
+      "kl": 0.030433058738708496,
+      "learning_rate": 9.399999999999999e-07,
+      "logits/chosen": -50427842.43037975,
+      "logits/rejected": -49013747.358024694,
+      "logps/chosen": -314.9666732594937,
+      "logps/rejected": -207.13416280864197,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49998149275779724,
+      "loss/total_with_kl": 0.5002912878990173,
+      "metrics/advantage_var": 0.06193568930029869,
+      "regularization/advantage_var": 0.06193568930029869,
+      "regularization/kl": 0.00030967846396379173,
+      "rewards/chosen": 0.00026683563578732404,
+      "rewards/margins": 0.00017124155932459015,
+      "rewards/rejected": 9.55940764627339e-05,
+      "stability/repetition_rate_mean": 0.20546108484268188,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 31.1264705657959,
+      "stability/response_length_var": 968.8571166992188,
+      "stability/token_entropy_mean": 3.6875,
+      "step": 95
+    },
+    {
+      "epoch": 0.5118362124120281,
+      "grad_norm": 0.11534037441015244,
+      "kl": 0.03825628012418747,
+      "learning_rate": 9.9e-07,
+      "logits/chosen": -54771206.91891892,
+      "logits/rejected": -50969665.488372095,
+      "logps/chosen": -359.78301836993245,
+      "logps/rejected": -216.57428688226744,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998970925807953,
+      "loss/total_with_kl": 0.5002375841140747,
+      "metrics/advantage_var": 0.06808135658502579,
+      "regularization/advantage_var": 0.06808135658502579,
+      "regularization/kl": 0.00034040678292512894,
+      "rewards/chosen": 0.0008716205688747199,
+      "rewards/margins": 0.000825567181953267,
+      "rewards/rejected": 4.605338692145292e-05,
+      "stability/repetition_rate_mean": 0.23228970170021057,
+      "stability/response_length_mean": 37.875,
+      "stability/response_length_std": 30.248672485351562,
+      "stability/response_length_var": 914.9821166992188,
+      "stability/token_entropy_mean": 4.19140625,
+      "step": 100
+    },
+    {
+      "epoch": 0.5374280230326296,
+      "grad_norm": 0.11625107377767563,
+      "kl": 0.038600411266088486,
+      "learning_rate": 9.999815554351224e-07,
+      "logits/chosen": -59427038.072289154,
+      "logits/rejected": -53858540.051948056,
+      "logps/chosen": -399.85146837349396,
+      "logps/rejected": -209.96349939123377,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999867379665375,
+      "loss/total_with_kl": 0.5004057884216309,
+      "metrics/advantage_var": 0.0838162824511528,
+      "regularization/advantage_var": 0.0838162824511528,
+      "regularization/kl": 0.0004190813924651593,
+      "rewards/chosen": 0.0005104481186493333,
+      "rewards/margins": 0.00010723205834995542,
+      "rewards/rejected": 0.00040321606029937793,
+      "stability/repetition_rate_mean": 0.2053571343421936,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 29.89505386352539,
+      "stability/response_length_var": 893.7142944335938,
+      "stability/token_entropy_mean": 3.41796875,
+      "step": 105
+    },
+    {
+      "epoch": 0.5630198336532309,
+      "grad_norm": 0.09513776749372482,
+      "kl": 0.034523267298936844,
+      "learning_rate": 9.999066267225444e-07,
+      "logits/chosen": -57464671.61445783,
+      "logits/rejected": -53675948.883116886,
+      "logps/chosen": -378.51877823795184,
+      "logps/rejected": -206.05215097402598,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999869465827942,
+      "loss/total_with_kl": 0.5003308653831482,
+      "metrics/advantage_var": 0.06877034902572632,
+      "regularization/advantage_var": 0.06877034902572632,
+      "regularization/kl": 0.00034385171602480114,
+      "rewards/chosen": 0.0004449200827673257,
+      "rewards/margins": 8.504402561005693e-05,
+      "rewards/rejected": 0.0003598760571572688,
+      "stability/repetition_rate_mean": 0.321445494890213,
+      "stability/response_length_mean": 22.625,
+      "stability/response_length_std": 24.68624496459961,
+      "stability/response_length_var": 609.4107055664062,
+      "stability/token_entropy_mean": 4.3740234375,
+      "step": 110
+    },
+    {
+      "epoch": 0.5886116442738324,
+      "grad_norm": 0.08769083023071289,
+      "kl": 0.04661265388131142,
+      "learning_rate": 9.997740697079592e-07,
+      "logits/chosen": -60680289.215189874,
+      "logits/rejected": -53502097.38271605,
+      "logps/chosen": -364.5057357594937,
+      "logps/rejected": -201.1548514660494,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999448359012604,
+      "loss/total_with_kl": 0.5003185272216797,
+      "metrics/advantage_var": 0.07474813610315323,
+      "regularization/advantage_var": 0.07474813610315323,
+      "regularization/kl": 0.00037374068051576614,
+      "rewards/chosen": 0.0008680501693411718,
+      "rewards/margins": 0.000450884205235189,
+      "rewards/rejected": 0.0004171659641059828,
+      "stability/repetition_rate_mean": 0.3046875,
+      "stability/response_length_mean": 41.125,
+      "stability/response_length_std": 31.60215950012207,
+      "stability/response_length_var": 998.6964111328125,
+      "stability/token_entropy_mean": 4.880859375,
+      "step": 115
+    },
+    {
+      "epoch": 0.6142034548944337,
+      "grad_norm": 0.09087492525577545,
+      "kl": 0.02683141827583313,
+      "learning_rate": 9.995838996722914e-07,
+      "logits/chosen": -57850217.4117647,
+      "logits/rejected": -54498101.65269461,
+      "logps/chosen": -384.176087622549,
+      "logps/rejected": -212.20452844311376,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993181228637695,
+      "loss/total_with_kl": 0.5003178119659424,
+      "metrics/advantage_var": 0.07719477266073227,
+      "regularization/advantage_var": 0.07719477266073227,
+      "regularization/kl": 0.000385973893571645,
+      "rewards/chosen": 0.0006882845381506128,
+      "rewards/margins": 0.0005699411889836399,
+      "rewards/rejected": 0.00011834334916697291,
+      "stability/repetition_rate_mean": 0.18938209116458893,
+      "stability/response_length_mean": 29.375,
+      "stability/response_length_std": 29.548688888549805,
+      "stability/response_length_var": 873.125,
+      "stability/token_entropy_mean": 5.3125,
+      "step": 120
+    },
+    {
+      "epoch": 0.6397952655150352,
+      "grad_norm": 0.08841491490602493,
+      "kl": 0.03785186633467674,
+      "learning_rate": 9.993361385379874e-07,
+      "logits/chosen": -60808739.06849315,
+      "logits/rejected": -55804009.93103448,
+      "logps/chosen": -417.3874946489726,
+      "logps/rejected": -216.35508710488506,
+      "loss": 0.5003,
+      "loss/base_kto": 0.5000187158584595,
+      "loss/total_with_kl": 0.500385046005249,
+      "metrics/advantage_var": 0.0732690691947937,
+      "regularization/advantage_var": 0.0732690691947937,
+      "regularization/kl": 0.0003663453389890492,
+      "rewards/chosen": -8.181337751958468e-05,
+      "rewards/margins": -0.00020296169137557255,
+      "rewards/rejected": 0.00012114831385598786,
+      "stability/repetition_rate_mean": 0.15937499701976776,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 24.93276596069336,
+      "stability/response_length_var": 621.6428833007812,
+      "stability/token_entropy_mean": 3.63671875,
+      "step": 125
+    },
+    {
+      "epoch": 0.6653870761356366,
+      "grad_norm": 0.09630157798528671,
+      "kl": 0.044555749744176865,
+      "learning_rate": 9.99030814866488e-07,
+      "logits/chosen": -57510337.23870968,
+      "logits/rejected": -53975995.733333334,
+      "logps/chosen": -392.6136340725806,
+      "logps/rejected": -183.91416903409092,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49994951486587524,
+      "loss/total_with_kl": 0.5003226399421692,
+      "metrics/advantage_var": 0.0746237188577652,
+      "regularization/advantage_var": 0.0746237188577652,
+      "regularization/kl": 0.000373118557035923,
+      "rewards/chosen": 0.00044304853485476587,
+      "rewards/margins": 0.00039567723542539255,
+      "rewards/rejected": 4.737129942937331e-05,
+      "stability/repetition_rate_mean": 0.23289062082767487,
+      "stability/response_length_mean": 43.25,
+      "stability/response_length_std": 29.620214462280273,
+      "stability/response_length_var": 877.3571166992188,
+      "stability/token_entropy_mean": 3.642578125,
+      "step": 130
+    },
+    {
+      "epoch": 0.690978886756238,
+      "grad_norm": 0.10103689879179001,
+      "kl": 0.04840748757123947,
+      "learning_rate": 9.986679638549358e-07,
+      "logits/chosen": -59991988.61349693,
+      "logits/rejected": -55199457.01910828,
+      "logps/chosen": -401.72205713190186,
+      "logps/rejected": -247.8357882165605,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999224841594696,
+      "loss/total_with_kl": 0.5003660321235657,
+      "metrics/advantage_var": 0.08870525658130646,
+      "regularization/advantage_var": 0.08870525658130646,
+      "regularization/kl": 0.00044352628174237907,
+      "rewards/chosen": 0.0009412271844828787,
+      "rewards/margins": 0.000591533045224002,
+      "rewards/rejected": 0.00034969413925887673,
+      "stability/repetition_rate_mean": 0.2230902761220932,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 30.20406723022461,
+      "stability/response_length_var": 912.2857055664062,
+      "stability/token_entropy_mean": 3.40625,
+      "step": 135
+    },
+    {
+      "epoch": 0.7165706973768394,
+      "grad_norm": 0.12567856907844543,
+      "kl": 0.03631413355469704,
+      "learning_rate": 9.982476273321175e-07,
+      "logits/chosen": -57247570.23030303,
+      "logits/rejected": -50254762.11612903,
+      "logps/chosen": -367.1258049242424,
+      "logps/rejected": -190.0154485887097,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993759393692017,
+      "loss/total_with_kl": 0.5003074407577515,
+      "metrics/advantage_var": 0.0739670991897583,
+      "regularization/advantage_var": 0.0739670991897583,
+      "regularization/kl": 0.00036983549944125116,
+      "rewards/chosen": 0.000639494214997147,
+      "rewards/margins": 0.0004491325897526065,
+      "rewards/rejected": 0.00019036162524454055,
+      "stability/repetition_rate_mean": 0.359375,
+      "stability/response_length_mean": 56.25,
+      "stability/response_length_std": 21.920310974121094,
+      "stability/response_length_var": 480.5,
+      "stability/token_entropy_mean": 4.75,
+      "step": 140
+    },
+    {
+      "epoch": 0.7421625079974408,
+      "grad_norm": 0.09891662001609802,
+      "kl": 0.056184422224760056,
+      "learning_rate": 9.977698537536417e-07,
+      "logits/chosen": -51311412.43373494,
+      "logits/rejected": -51875494.233766235,
+      "logps/chosen": -330.2195500753012,
+      "logps/rejected": -199.49284699675326,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49993544816970825,
+      "loss/total_with_kl": 0.5002543926239014,
+      "metrics/advantage_var": 0.06378551572561264,
+      "regularization/advantage_var": 0.06378551572561264,
+      "regularization/kl": 0.00031892754486761987,
+      "rewards/chosen": 0.0005227750144809125,
+      "rewards/margins": 0.0005721743595297488,
+      "rewards/rejected": -4.93993450488363e-05,
+      "stability/repetition_rate_mean": 0.24448327720165253,
+      "stability/response_length_mean": 28.875,
+      "stability/response_length_std": 29.73423957824707,
+      "stability/response_length_var": 884.125,
+      "stability/token_entropy_mean": 5.236328125,
+      "step": 145
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.10929251462221146,
+      "kl": 0.05055147036910057,
+      "learning_rate": 9.972346981963546e-07,
+      "logits/chosen": -55581629.74117647,
+      "logits/rejected": -50397125.97333334,
+      "logps/chosen": -397.8387867647059,
+      "logps/rejected": -207.7371484375,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998582899570465,
+      "loss/total_with_kl": 0.5003008842468262,
+      "metrics/advantage_var": 0.08850937336683273,
+      "regularization/advantage_var": 0.08850937336683273,
+      "regularization/kl": 0.00044254687963984907,
+      "rewards/chosen": 0.0009563709883128895,
+      "rewards/margins": 0.001068605093979368,
+      "rewards/rejected": -0.00011223410566647848,
+      "stability/repetition_rate_mean": 0.2115885466337204,
+      "stability/response_length_mean": 28.25,
+      "stability/response_length_std": 29.90341567993164,
+      "stability/response_length_var": 894.2142944335938,
+      "stability/token_entropy_mean": 4.80859375,
+      "step": 150
+    },
+    {
+      "epoch": 0.7933461292386437,
+      "grad_norm": 0.09307374805212021,
+      "kl": 0.048836734145879745,
+      "learning_rate": 9.966422223519885e-07,
+      "logits/chosen": -54227281.31764706,
+      "logits/rejected": -49584203.093333334,
+      "logps/chosen": -360.93809742647056,
+      "logps/rejected": -191.300625,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999396502971649,
+      "loss/total_with_kl": 0.5003570318222046,
+      "metrics/advantage_var": 0.08347625285387039,
+      "regularization/advantage_var": 0.08347625285387039,
+      "regularization/kl": 0.00041738126310519874,
+      "rewards/chosen": 0.0005118872312938465,
+      "rewards/margins": 0.00046203332277489637,
+      "rewards/rejected": 4.985390851895015e-05,
+      "stability/repetition_rate_mean": 0.3074128031730652,
+      "stability/response_length_mean": 61.375,
+      "stability/response_length_std": 7.424621105194092,
+      "stability/response_length_var": 55.125,
+      "stability/token_entropy_mean": 4.6044921875,
+      "step": 155
+    },
+    {
+      "epoch": 0.818937939859245,
+      "grad_norm": 0.09413865953683853,
+      "kl": 0.0488591343164444,
+      "learning_rate": 9.959924945200523e-07,
+      "logits/chosen": -55468525.71428572,
+      "logits/rejected": -49233495.578947365,
+      "logps/chosen": -416.8404947916667,
+      "logps/rejected": -186.8857293379934,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999455511569977,
+      "loss/total_with_kl": 0.5003138780593872,
+      "metrics/advantage_var": 0.07364356517791748,
+      "regularization/advantage_var": 0.07364356517791748,
+      "regularization/kl": 0.00036821779212914407,
+      "rewards/chosen": 0.0005480412039018813,
+      "rewards/margins": 0.0004319082198659878,
+      "rewards/rejected": 0.00011613298403589349,
+      "stability/repetition_rate_mean": 0.291015625,
+      "stability/response_length_mean": 26.25,
+      "stability/response_length_std": 31.535694122314453,
+      "stability/response_length_var": 994.5,
+      "stability/token_entropy_mean": 3.7998046875,
+      "step": 160
+    },
+    {
+      "epoch": 0.8445297504798465,
+      "grad_norm": 0.10981868207454681,
+      "kl": 0.04979108273983002,
+      "learning_rate": 9.952855895999568e-07,
+      "logits/chosen": -54661708.505747125,
+      "logits/rejected": -50090075.17808219,
+      "logps/chosen": -370.8274066091954,
+      "logps/rejected": -212.58053296232876,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999002516269684,
+      "loss/total_with_kl": 0.5002889633178711,
+      "metrics/advantage_var": 0.07773963361978531,
+      "regularization/advantage_var": 0.07773963361978531,
+      "regularization/kl": 0.0003886981576215476,
+      "rewards/chosen": 0.0008934761943488285,
+      "rewards/margins": 0.0007776661556483329,
+      "rewards/rejected": 0.00011581003870049568,
+      "stability/repetition_rate_mean": 0.2632460594177246,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 23.150131225585938,
+      "stability/response_length_var": 535.9285888671875,
+      "stability/token_entropy_mean": 4.81640625,
+      "step": 165
+    },
+    {
+      "epoch": 0.8701215611004478,
+      "grad_norm": 0.08388755470514297,
+      "kl": 0.06272337585687637,
+      "learning_rate": 9.9452158908238e-07,
+      "logits/chosen": -59387522.27218935,
+      "logits/rejected": -52949961.74834437,
+      "logps/chosen": -441.2528661242604,
+      "logps/rejected": -191.2500776076159,
+      "loss": 0.5003,
+      "loss/base_kto": 0.49989959597587585,
+      "loss/total_with_kl": 0.5002941489219666,
+      "metrics/advantage_var": 0.07889693975448608,
+      "regularization/advantage_var": 0.07889693975448608,
+      "regularization/kl": 0.0003944846976082772,
+      "rewards/chosen": 0.000889729113268429,
+      "rewards/margins": 0.0008121537112671523,
+      "rewards/rejected": 7.757540200127671e-05,
+      "stability/repetition_rate_mean": 0.2809279263019562,
+      "stability/response_length_mean": 28.25,
+      "stability/response_length_std": 30.844078063964844,
+      "stability/response_length_var": 951.3571166992188,
+      "stability/token_entropy_mean": 3.09033203125,
+      "step": 170
+    },
+    {
+      "epoch": 0.8957133717210493,
+      "grad_norm": 0.09992633014917374,
+      "kl": 0.06644636392593384,
+      "learning_rate": 9.937005810398745e-07,
+      "logits/chosen": -56929020.915662654,
+      "logits/rejected": -48413912.103896104,
+      "logps/chosen": -356.35577466114455,
+      "logps/rejected": -194.76746398133116,
+      "loss": 0.5003,
+      "loss/base_kto": 0.499949187040329,
+      "loss/total_with_kl": 0.5003606677055359,
+      "metrics/advantage_var": 0.08229832351207733,
+      "regularization/advantage_var": 0.08229832351207733,
+      "regularization/kl": 0.0004114916082471609,
+      "rewards/chosen": 0.0008141423564359367,
+      "rewards/margins": 0.0004135140424331689,
+      "rewards/rejected": 0.0004006283140027678,
+      "stability/repetition_rate_mean": 0.28187286853790283,
+      "stability/response_length_mean": 32.5,
+      "stability/response_length_std": 27.969371795654297,
+      "stability/response_length_var": 782.2857055664062,
+      "stability/token_entropy_mean": 4.7412109375,
+      "step": 175
+    },
+    {
+      "epoch": 0.9213051823416507,
+      "grad_norm": 0.09833401441574097,
+      "kl": 0.0622967854142189,
+      "learning_rate": 9.928226601167138e-07,
+      "logits/chosen": -52938997.00613497,
+      "logits/rejected": -48348688.30573248,
+      "logps/chosen": -376.0574194785276,
+      "logps/rejected": -211.23741042993632,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999646246433258,
+      "loss/total_with_kl": 0.500411868095398,
+      "metrics/advantage_var": 0.08944838494062424,
+      "regularization/advantage_var": 0.08944838494062424,
+      "regularization/kl": 0.00044724190956912935,
+      "rewards/chosen": 0.0008839887161196375,
+      "rewards/margins": 0.00027612949146912696,
+      "rewards/rejected": 0.0006078592246505105,
+      "stability/repetition_rate_mean": 0.16015625,
+      "stability/response_length_mean": 40.375,
+      "stability/response_length_std": 32.61874008178711,
+      "stability/response_length_var": 1063.982177734375,
+      "stability/token_entropy_mean": 3.53125,
+      "step": 180
+    },
+    {
+      "epoch": 0.946896992962252,
+      "grad_norm": 0.09617482125759125,
+      "kl": 0.0709846243262291,
+      "learning_rate": 9.918879275179817e-07,
+      "logits/chosen": -54943105.58024691,
+      "logits/rejected": -49885858.025316454,
+      "logps/chosen": -311.92582947530866,
+      "logps/rejected": -197.01087816455697,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4999426007270813,
+      "loss/total_with_kl": 0.5003011226654053,
+      "metrics/advantage_var": 0.07169583439826965,
+      "regularization/advantage_var": 0.07169583439826965,
+      "regularization/kl": 0.0003584791556932032,
+      "rewards/chosen": 0.0007554439850795417,
+      "rewards/margins": 0.0004727014776067187,
+      "rewards/rejected": 0.00028274250747282293,
+      "stability/repetition_rate_mean": 0.18998579680919647,
+      "stability/response_length_mean": 22.25,
+      "stability/response_length_std": 27.78874397277832,
+      "stability/response_length_var": 772.2142944335938,
+      "stability/token_entropy_mean": 3.828125,
+      "step": 185
+    },
+    {
+      "epoch": 0.9724888035828535,
+      "grad_norm": 0.10346374660730362,
+      "kl": 0.05659087374806404,
+      "learning_rate": 9.90896490997906e-07,
+      "logits/chosen": -53424790.22754491,
+      "logits/rejected": -50103520.20915033,
+      "logps/chosen": -392.2973194236527,
+      "logps/rejected": -211.16874744689542,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998948276042938,
+      "loss/total_with_kl": 0.5002766847610474,
+      "metrics/advantage_var": 0.07637263834476471,
+      "regularization/advantage_var": 0.07637263834476471,
+      "regularization/kl": 0.0003818631812464446,
+      "rewards/chosen": 0.0010801159336181459,
+      "rewards/margins": 0.000781740584588929,
+      "rewards/rejected": 0.0002983753490292169,
+      "stability/repetition_rate_mean": 0.1047247052192688,
+      "stability/response_length_mean": 11.375,
+      "stability/response_length_std": 21.738296508789062,
+      "stability/response_length_var": 472.5535583496094,
+      "stability/token_entropy_mean": 5.38671875,
+      "step": 190
+    },
+    {
+      "epoch": 0.9980806142034548,
+      "grad_norm": 0.10313382744789124,
+      "kl": 0.055899590253829956,
+      "learning_rate": 9.898484648474362e-07,
+      "logits/chosen": -60530562.477419354,
+      "logits/rejected": -54311222.303030305,
+      "logps/chosen": -392.6547379032258,
+      "logps/rejected": -197.9125946969697,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4998457431793213,
+      "loss/total_with_kl": 0.5002177357673645,
+      "metrics/advantage_var": 0.07440383732318878,
+      "regularization/advantage_var": 0.07440383732318878,
+      "regularization/kl": 0.00037201918894425035,
+      "rewards/chosen": 0.0013020292405159244,
+      "rewards/margins": 0.0012345370724578758,
+      "rewards/rejected": 6.74921680580486e-05,
+      "stability/repetition_rate_mean": 0.3515625,
+      "stability/response_length_mean": 40.875,
+      "stability/response_length_std": 31.96175193786621,
+      "stability/response_length_var": 1021.5535888671875,
+      "stability/token_entropy_mean": 4.490234375,
+      "step": 195
+    },
+    {
+      "epoch": 1.0,
+      "step": 196,
+      "total_flos": 3.1861866257511875e+18,
+      "train_loss": 0.5003062401499067,
+      "train_runtime": 3152.1109,
+      "train_samples_per_second": 31.725,
+      "train_steps_per_second": 0.062
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 196,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.1861866257511875e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

mistral7b_kto_kl-beta0.01/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85beda43fa0c145c1439cc242873864caf2fddd3411f37e00a57d3806e34bafa
+size 8017