Upload folder using huggingface_hub

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/README.md

@@ -0,0 +1,64 @@
+---
+library_name: peft
+license: other
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- base_model:adapter:mistralai/Mistral-7B-v0.1
+- llama-factory
+- lora
+- transformers
+pipeline_tag: text-generation
+model-index:
+- name: mistral7b_kto_mmd_rho0.001_lambda_kr0.01
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral7b_kto_mmd_rho0.001_lambda_kr0.01
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the shp_kto_train dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 512
+- total_eval_batch_size: 64
+- optimizer: Use adamw_torch_fused with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 1.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.17.1
+- Transformers 4.57.1
+- Pytorch 2.10.0+cu128
+- Datasets 4.0.0
+- Tokenizers 0.22.2

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:823028f3b1b88c833e716310de2da0f11c38588c618204a8e35b71771f03937c
+size 42002584

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.1861866257511875e+18,
+    "train_loss": 0.500500318955402,
+    "train_runtime": 3156.0327,
+    "train_samples_per_second": 31.685,
+    "train_steps_per_second": 0.062
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/chat_template.jinja

@@ -0,0 +1,3 @@
+{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 and system_message is defined %}{% set content = system_message + '
+
+' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + '[/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content + '</s>' }}{% endif %}{% endfor %}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/README.md

@@ -0,0 +1,208 @@
+---
+base_model: mistralai/Mistral-7B-v0.1
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:mistralai/Mistral-7B-v0.1
+- llama-factory
+- lora
+- transformers
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.17.1

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/adapter_config.json

@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:823028f3b1b88c833e716310de2da0f11c38588c618204a8e35b71771f03937c
+size 42002584

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/chat_template.jinja

@@ -0,0 +1,3 @@
+{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 and system_message is defined %}{% set content = system_message + '
+
+' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + '[/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content + '</s>' }}{% endif %}{% endfor %}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c07c143c3ee7b471084b703492760bc1e994544e03afdfedfa1f65cf598d4848
+size 31467333

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a88b090cad587dea4da18350396e09a256976278b0c94bb6e429f617ef4d511
+size 31467333

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c903aa3bb603f2299f21175412c82226fcaaf2df36565f204fc52f963163cbf2
+size 31467333

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d935d438734fca2c51de0ea72d5d352f9a41dc2c7bfcfb4eae03456ac78f5e2
+size 31467397

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d613e1e1536df63a5a12695abe06c23e834b87cf270647755493bc9891cc14
+size 31467397

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9fee4ac468c8adc18325b26fb7267494f4757ac223abee5a5b8d1f83fc2046cf
+size 31467397

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4546dbed6d94f3c424d565badd4332ce0c0f274298dbdb127a915aeefae22559
+size 31467397

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40c3b5369000e5ad8985817f4cccc2a93b7e3df51ce2206983e2c726484bec39
+size 31467397

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/global_step196/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d71a013e399ff7b08e890b748b7e4d674ed042a8e9ab786f7b2ebe03f39a58a1
+size 42288409

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/latest

@@ -0,0 +1 @@
+global_step196

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7efbc3b39e557ed047a255851058c3d62216b6bc5794c19289879cf099f52035
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce29a8767a7d907dd24987aa2c3e654d4317f3042fbc13b5b72cadb46d43311a
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61a48db011646b4e9a867bf12f4a233cad5dfbfe309686f8996c250196d3783a
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9562ee822472a4f01dcd6349ab3d1ef42a48915fe3b92e843a0c37db53c8421
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7d2767d83c3bf27f12db022b0632e2c4f8c164274ba75e380cf18f9d5f21819
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76816358d4e5db8149d60d55234db658d67a13c0c1ce05d7404cf7125a676a5c
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1562e7520c977d178183d641f70abcf3f57da2489938756cfbebf9b6e6c1a9fd
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6b6cabaed045c5398cd1b732f7ec48bd363f3b43cd24e0e70e641a42bd00c28
+size 16389

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83820952e8eb59075696a2d6e1f7ef27d70c5576cb2781806606f38f86685d3e
+size 1401

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/trainer_state.json

@@ -0,0 +1,965 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 196,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.025591810620601407,
+      "grad_norm": 0.5721998810768127,
+      "kl": 0.008188384585082531,
+      "learning_rate": 4e-08,
+      "logits/chosen": -59712215.578947365,
+      "logits/rejected": -50420742.0952381,
+      "logps/chosen": -438.99403782894734,
+      "logps/rejected": -198.40638950892858,
+      "loss": 0.5009,
+      "loss/base_kto": 0.4994224011898041,
+      "metrics/advantage_var": 0.04372745379805565,
+      "regularization/mmd": 0.0012165767839178443,
+      "regularization/rkhs_norm": 1.2165768146514893,
+      "rewards/chosen": 0.005160976397363763,
+      "rewards/margins": 0.004860207811931619,
+      "rewards/rejected": 0.00030076858543214344,
+      "step": 5
+    },
+    {
+      "epoch": 0.05118362124120281,
+      "grad_norm": 0.6033456921577454,
+      "kl": 0.017390631139278412,
+      "learning_rate": 9e-08,
+      "logits/chosen": -59291993.766233765,
+      "logits/rejected": -50017332.43373494,
+      "logps/chosen": -429.9233969155844,
+      "logps/rejected": -192.87807087725903,
+      "loss": 0.5018,
+      "loss/base_kto": 0.49968767166137695,
+      "metrics/advantage_var": 0.07829762995243073,
+      "regularization/mmd": 0.0020083182025700808,
+      "regularization/rkhs_norm": 2.0083181858062744,
+      "rewards/chosen": 0.001513990101876197,
+      "rewards/margins": 0.0025270523678583077,
+      "rewards/rejected": -0.0010130622659821108,
+      "stability/repetition_rate_mean": 0.3570815920829773,
+      "stability/response_length_mean": 46.125,
+      "stability/response_length_std": 27.231481552124023,
+      "stability/response_length_var": 741.5535888671875,
+      "stability/token_entropy_mean": 3.29638671875,
+      "step": 10
+    },
+    {
+      "epoch": 0.07677543186180422,
+      "grad_norm": 0.6098381876945496,
+      "kl": 0.00862761028110981,
+      "learning_rate": 1.4e-07,
+      "logits/chosen": -56235380.36363637,
+      "logits/rejected": -51927694.03870968,
+      "logps/chosen": -404.1193181818182,
+      "logps/rejected": -200.67879284274193,
+      "loss": 0.5016,
+      "loss/base_kto": 0.5000181198120117,
+      "metrics/advantage_var": 0.05759938433766365,
+      "regularization/mmd": 0.0016775511903688312,
+      "regularization/rkhs_norm": 1.6775509119033813,
+      "rewards/chosen": 0.0011517706242474644,
+      "rewards/margins": -0.0003764610311502584,
+      "rewards/rejected": 0.0015282316553977228,
+      "stability/repetition_rate_mean": 0.2651909589767456,
+      "stability/response_length_mean": 27.125,
+      "stability/response_length_std": 30.79163932800293,
+      "stability/response_length_var": 948.125,
+      "stability/token_entropy_mean": 3.82421875,
+      "step": 15
+    },
+    {
+      "epoch": 0.10236724248240563,
+      "grad_norm": 0.6189659237861633,
+      "kl": 0.011922934092581272,
+      "learning_rate": 1.8999999999999998e-07,
+      "logits/chosen": -51664051.2,
+      "logits/rejected": -48054336.0,
+      "logps/chosen": -343.67216796875,
+      "logps/rejected": -225.6396484375,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5000166296958923,
+      "metrics/advantage_var": 0.05863838270306587,
+      "regularization/mmd": 0.001691564917564392,
+      "regularization/rkhs_norm": 1.691564917564392,
+      "rewards/chosen": 0.002454766258597374,
+      "rewards/margins": 8.368939161300685e-05,
+      "rewards/rejected": 0.0023710768669843673,
+      "stability/repetition_rate_mean": 0.2155257910490036,
+      "stability/response_length_mean": 44.0,
+      "stability/response_length_std": 28.0611572265625,
+      "stability/response_length_var": 787.4285888671875,
+      "stability/token_entropy_mean": 4.42578125,
+      "step": 20
+    },
+    {
+      "epoch": 0.12795905310300704,
+      "grad_norm": 0.5778173804283142,
+      "kl": 0.018707742914557457,
+      "learning_rate": 2.4e-07,
+      "logits/chosen": -56476908.57931034,
+      "logits/rejected": -52024659.382857144,
+      "logps/chosen": -378.4651400862069,
+      "logps/rejected": -202.7953794642857,
+      "loss": 0.5019,
+      "loss/base_kto": 0.4996436536312103,
+      "metrics/advantage_var": 0.0549849271774292,
+      "regularization/mmd": 0.0017472734907642007,
+      "regularization/rkhs_norm": 1.7472734451293945,
+      "rewards/chosen": 0.0026301628556744806,
+      "rewards/margins": 0.0028841727326068974,
+      "rewards/rejected": -0.00025400987693241663,
+      "stability/repetition_rate_mean": 0.17606504261493683,
+      "stability/response_length_mean": 43.375,
+      "stability/response_length_std": 28.54538917541504,
+      "stability/response_length_var": 814.8392944335938,
+      "stability/token_entropy_mean": 4.763671875,
+      "step": 25
+    },
+    {
+      "epoch": 0.15355086372360843,
+      "grad_norm": 0.6033927202224731,
+      "kl": 0.017847830429673195,
+      "learning_rate": 2.9e-07,
+      "logits/chosen": -57730419.11409396,
+      "logits/rejected": -51521314.43274854,
+      "logps/chosen": -468.5690016778523,
+      "logps/rejected": -198.18969298245614,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5006142854690552,
+      "metrics/advantage_var": 0.08329301327466965,
+      "regularization/mmd": 0.0019578204955905676,
+      "regularization/rkhs_norm": 1.9578202962875366,
+      "rewards/chosen": 0.0012253856898954251,
+      "rewards/margins": -0.004593525076170741,
+      "rewards/rejected": 0.005818910766066166,
+      "stability/repetition_rate_mean": 0.1544828861951828,
+      "stability/response_length_mean": 43.0,
+      "stability/response_length_std": 29.602123260498047,
+      "stability/response_length_var": 876.2857055664062,
+      "stability/token_entropy_mean": 4.552734375,
+      "step": 30
+    },
+    {
+      "epoch": 0.17914267434420986,
+      "grad_norm": 0.6821840405464172,
+      "kl": 0.021474814042448997,
+      "learning_rate": 3.4000000000000003e-07,
+      "logits/chosen": -57633720.70886076,
+      "logits/rejected": -53879119.01234568,
+      "logps/chosen": -385.90676918512656,
+      "logps/rejected": -195.13110050154322,
+      "loss": 0.5017,
+      "loss/base_kto": 0.4995640218257904,
+      "metrics/advantage_var": 0.06586408615112305,
+      "regularization/mmd": 0.0018217876786366105,
+      "regularization/rkhs_norm": 1.821787714958191,
+      "rewards/chosen": 0.0029728393765944467,
+      "rewards/margins": 0.0034354667138971375,
+      "rewards/rejected": -0.0004626273373026907,
+      "stability/repetition_rate_mean": 0.34880331158638,
+      "stability/response_length_mean": 53.375,
+      "stability/response_length_std": 21.14533042907715,
+      "stability/response_length_var": 447.125,
+      "stability/token_entropy_mean": 5.326171875,
+      "step": 35
+    },
+    {
+      "epoch": 0.20473448496481125,
+      "grad_norm": 0.5855079293251038,
+      "kl": 0.018473921343684196,
+      "learning_rate": 3.8999999999999997e-07,
+      "logits/chosen": -56637171.2,
+      "logits/rejected": -50217504.0,
+      "logps/chosen": -421.93681640625,
+      "logps/rejected": -219.247998046875,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5011991858482361,
+      "metrics/advantage_var": 0.08207302540540695,
+      "regularization/mmd": 0.0019648734014481306,
+      "regularization/rkhs_norm": 1.9648735523223877,
+      "rewards/chosen": -0.003416117653250694,
+      "rewards/margins": -0.009540746733546258,
+      "rewards/rejected": 0.006124629080295563,
+      "stability/repetition_rate_mean": 0.142578125,
+      "stability/response_length_mean": 33.125,
+      "stability/response_length_std": 33.01704025268555,
+      "stability/response_length_var": 1090.125,
+      "stability/token_entropy_mean": 4.9765625,
+      "step": 40
+    },
+    {
+      "epoch": 0.23032629558541268,
+      "grad_norm": 0.6406768560409546,
+      "kl": 0.032560475170612335,
+      "learning_rate": 4.3999999999999997e-07,
+      "logits/chosen": -52695545.12751678,
+      "logits/rejected": -50770458.94736842,
+      "logps/chosen": -352.29299496644296,
+      "logps/rejected": -243.82858187134502,
+      "loss": 0.5017,
+      "loss/base_kto": 0.5001169443130493,
+      "metrics/advantage_var": 0.062119271606206894,
+      "regularization/mmd": 0.0017398166237398982,
+      "regularization/rkhs_norm": 1.739816665649414,
+      "rewards/chosen": 0.0035465471696533614,
+      "rewards/margins": -0.0010111425584421558,
+      "rewards/rejected": 0.004557689728095517,
+      "stability/repetition_rate_mean": 0.1013549193739891,
+      "stability/response_length_mean": 29.25,
+      "stability/response_length_std": 29.51875877380371,
+      "stability/response_length_var": 871.3571166992188,
+      "stability/token_entropy_mean": 4.708984375,
+      "step": 45
+    },
+    {
+      "epoch": 0.2559181062060141,
+      "grad_norm": 0.5807592868804932,
+      "kl": 0.04730672389268875,
+      "learning_rate": 4.9e-07,
+      "logits/chosen": -54038620.77852349,
+      "logits/rejected": -48295651.55555555,
+      "logps/chosen": -375.1285654362416,
+      "logps/rejected": -180.20700840643275,
+      "loss": 0.5015,
+      "loss/base_kto": 0.4994547963142395,
+      "metrics/advantage_var": 0.06544163823127747,
+      "regularization/mmd": 0.0017960931872949004,
+      "regularization/rkhs_norm": 1.7960929870605469,
+      "rewards/chosen": 0.007055705025692114,
+      "rewards/margins": 0.0036947803606983817,
+      "rewards/rejected": 0.0033609246649937323,
+      "stability/repetition_rate_mean": 0.4108419120311737,
+      "stability/response_length_mean": 55.375,
+      "stability/response_length_std": 16.361431121826172,
+      "stability/response_length_var": 267.6964416503906,
+      "stability/token_entropy_mean": 3.814453125,
+      "step": 50
+    },
+    {
+      "epoch": 0.28150991682661547,
+      "grad_norm": 0.6596181988716125,
+      "kl": 0.05266108736395836,
+      "learning_rate": 5.4e-07,
+      "logits/chosen": -56956718.111801244,
+      "logits/rejected": -49555314.31446541,
+      "logps/chosen": -344.4710064052795,
+      "logps/rejected": -230.38991253930817,
+      "loss": 0.5015,
+      "loss/base_kto": 0.5002086162567139,
+      "metrics/advantage_var": 0.06512904167175293,
+      "regularization/mmd": 0.001749949180521071,
+      "regularization/rkhs_norm": 1.7499490976333618,
+      "rewards/chosen": 0.004102396668854707,
+      "rewards/margins": -0.0017093748379286402,
+      "rewards/rejected": 0.005811771506783347,
+      "stability/repetition_rate_mean": 0.10613363981246948,
+      "stability/response_length_mean": 22.375,
+      "stability/response_length_std": 30.236862182617188,
+      "stability/response_length_var": 914.2678833007812,
+      "stability/token_entropy_mean": 4.5703125,
+      "step": 55
+    },
+    {
+      "epoch": 0.30710172744721687,
+      "grad_norm": 0.5826641917228699,
+      "kl": 0.050893235951662064,
+      "learning_rate": 5.9e-07,
+      "logits/chosen": -52669142.10909091,
+      "logits/rejected": -47815765.88387097,
+      "logps/chosen": -359.6506628787879,
+      "logps/rejected": -178.1343245967742,
+      "loss": 0.5015,
+      "loss/base_kto": 0.5002776384353638,
+      "metrics/advantage_var": 0.07838872820138931,
+      "regularization/mmd": 0.0018729189177975059,
+      "regularization/rkhs_norm": 1.872918963432312,
+      "rewards/chosen": 0.007016492612434156,
+      "rewards/margins": -0.0022417499639188557,
+      "rewards/rejected": 0.009258242576353012,
+      "stability/repetition_rate_mean": 0.20835673809051514,
+      "stability/response_length_mean": 51.125,
+      "stability/response_length_std": 21.90523338317871,
+      "stability/response_length_var": 479.83929443359375,
+      "stability/token_entropy_mean": 5.353515625,
+      "step": 60
+    },
+    {
+      "epoch": 0.3326935380678183,
+      "grad_norm": 0.5758509039878845,
+      "kl": 0.06566640734672546,
+      "learning_rate": 6.4e-07,
+      "logits/chosen": -57555868.90322581,
+      "logits/rejected": -47274523.92727273,
+      "logps/chosen": -381.8256048387097,
+      "logps/rejected": -183.51074810606062,
+      "loss": 0.5013,
+      "loss/base_kto": 0.49938079714775085,
+      "metrics/advantage_var": 0.06294770538806915,
+      "regularization/mmd": 0.0016916670138016343,
+      "regularization/rkhs_norm": 1.6916671991348267,
+      "rewards/chosen": 0.012116812121483588,
+      "rewards/margins": 0.004956388380287447,
+      "rewards/rejected": 0.007160423741196141,
+      "stability/repetition_rate_mean": 0.18097305297851562,
+      "stability/response_length_mean": 17.875,
+      "stability/response_length_std": 23.485177993774414,
+      "stability/response_length_var": 551.5535888671875,
+      "stability/token_entropy_mean": 5.66796875,
+      "step": 65
+    },
+    {
+      "epoch": 0.3582853486884197,
+      "grad_norm": 0.5520736575126648,
+      "kl": 0.08278707414865494,
+      "learning_rate": 6.9e-07,
+      "logits/chosen": -52775096.980645165,
+      "logits/rejected": -49028474.56969697,
+      "logps/chosen": -378.99866431451613,
+      "logps/rejected": -220.35059185606062,
+      "loss": 0.5012,
+      "loss/base_kto": 0.4992857575416565,
+      "metrics/advantage_var": 0.06611553579568863,
+      "regularization/mmd": 0.001765405759215355,
+      "regularization/rkhs_norm": 1.7654056549072266,
+      "rewards/chosen": 0.01297938746790732,
+      "rewards/margins": 0.0058224533077325755,
+      "rewards/rejected": 0.007156934160174745,
+      "stability/repetition_rate_mean": 0.1875,
+      "stability/response_length_mean": 33.125,
+      "stability/response_length_std": 33.0300178527832,
+      "stability/response_length_var": 1090.982177734375,
+      "stability/token_entropy_mean": 5.765625,
+      "step": 70
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.6104229688644409,
+      "kl": 0.10445437580347061,
+      "learning_rate": 7.4e-07,
+      "logits/chosen": -49496012.10810811,
+      "logits/rejected": -47883255.069767445,
+      "logps/chosen": -314.14432010135135,
+      "logps/rejected": -228.81213662790697,
+      "loss": 0.5014,
+      "loss/base_kto": 0.4992406964302063,
+      "metrics/advantage_var": 0.07885061949491501,
+      "regularization/mmd": 0.0018627401441335678,
+      "regularization/rkhs_norm": 1.862740397453308,
+      "rewards/chosen": 0.012649115678426382,
+      "rewards/margins": 0.005379979937126021,
+      "rewards/rejected": 0.007269135741300361,
+      "stability/repetition_rate_mean": 0.06355167925357819,
+      "stability/response_length_mean": 22.75,
+      "stability/response_length_std": 31.07479476928711,
+      "stability/response_length_var": 965.6428833007812,
+      "stability/token_entropy_mean": 2.16015625,
+      "step": 75
+    },
+    {
+      "epoch": 0.4094689699296225,
+      "grad_norm": 0.6028388738632202,
+      "kl": 0.11964090168476105,
+      "learning_rate": 7.9e-07,
+      "logits/chosen": -58612227.39072847,
+      "logits/rejected": -53431647.43195266,
+      "logps/chosen": -403.8477597268212,
+      "logps/rejected": -200.98007581360946,
+      "loss": 0.5012,
+      "loss/base_kto": 0.4987204074859619,
+      "metrics/advantage_var": 0.07979276031255722,
+      "regularization/mmd": 0.0019349249778315425,
+      "regularization/rkhs_norm": 1.9349247217178345,
+      "rewards/chosen": 0.01823797131216289,
+      "rewards/margins": 0.010667078210724629,
+      "rewards/rejected": 0.007570893101438263,
+      "stability/repetition_rate_mean": 0.18697917461395264,
+      "stability/response_length_mean": 42.0,
+      "stability/response_length_std": 30.481843948364258,
+      "stability/response_length_var": 929.1428833007812,
+      "stability/token_entropy_mean": 5.25,
+      "step": 80
+    },
+    {
+      "epoch": 0.4350607805502239,
+      "grad_norm": 0.6117395758628845,
+      "kl": 0.12551236152648926,
+      "learning_rate": 8.399999999999999e-07,
+      "logits/chosen": -51474676.438709676,
+      "logits/rejected": -47137016.24242424,
+      "logps/chosen": -358.1219758064516,
+      "logps/rejected": -230.35598958333333,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49821534752845764,
+      "metrics/advantage_var": 0.09083787351846695,
+      "regularization/mmd": 0.0020756726153194904,
+      "regularization/rkhs_norm": 2.0756723880767822,
+      "rewards/chosen": 0.021053169619652534,
+      "rewards/margins": 0.014171496211375426,
+      "rewards/rejected": 0.006881673408277107,
+      "stability/repetition_rate_mean": 0.1269342005252838,
+      "stability/response_length_mean": 27.375,
+      "stability/response_length_std": 30.39707374572754,
+      "stability/response_length_var": 923.9821166992188,
+      "stability/token_entropy_mean": 4.91796875,
+      "step": 85
+    },
+    {
+      "epoch": 0.46065259117082535,
+      "grad_norm": 0.6396152973175049,
+      "kl": 0.16505120694637299,
+      "learning_rate": 8.9e-07,
+      "logits/chosen": -58417874.05128205,
+      "logits/rejected": -50577639.02439024,
+      "logps/chosen": -380.5598958333333,
+      "logps/rejected": -187.15162919207316,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49800893664360046,
+      "metrics/advantage_var": 0.09185568243265152,
+      "regularization/mmd": 0.002058969810605049,
+      "regularization/rkhs_norm": 2.058969736099243,
+      "rewards/chosen": 0.02295044293770423,
+      "rewards/margins": 0.015432298295269762,
+      "rewards/rejected": 0.007518144642434469,
+      "stability/repetition_rate_mean": 0.2901870310306549,
+      "stability/response_length_mean": 32.875,
+      "stability/response_length_std": 30.31000518798828,
+      "stability/response_length_var": 918.6964111328125,
+      "stability/token_entropy_mean": 3.001953125,
+      "step": 90
+    },
+    {
+      "epoch": 0.48624440179142675,
+      "grad_norm": 0.6152090430259705,
+      "kl": 0.16731511056423187,
+      "learning_rate": 9.399999999999999e-07,
+      "logits/chosen": -50385446.88607595,
+      "logits/rejected": -48970211.55555555,
+      "logps/chosen": -314.7583811313291,
+      "logps/rejected": -207.0643325617284,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49809834361076355,
+      "metrics/advantage_var": 0.12706856429576874,
+      "regularization/mmd": 0.0023521382827311754,
+      "regularization/rkhs_norm": 2.3521382808685303,
+      "rewards/chosen": 0.023500275008286102,
+      "rewards/margins": 0.015562433063061765,
+      "rewards/rejected": 0.007937841945224337,
+      "stability/repetition_rate_mean": 0.19764858484268188,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 31.1264705657959,
+      "stability/response_length_var": 968.8571166992188,
+      "stability/token_entropy_mean": 3.689453125,
+      "step": 95
+    },
+    {
+      "epoch": 0.5118362124120281,
+      "grad_norm": 0.6100143790245056,
+      "kl": 0.21206355094909668,
+      "learning_rate": 9.9e-07,
+      "logits/chosen": -54718024.64864865,
+      "logits/rejected": -50911166.511627905,
+      "logps/chosen": -359.5405933277027,
+      "logps/rejected": -216.3824945494186,
+      "loss": 0.5006,
+      "loss/base_kto": 0.49836379289627075,
+      "metrics/advantage_var": 0.13836942613124847,
+      "regularization/mmd": 0.0024804905988276005,
+      "regularization/rkhs_norm": 2.4804904460906982,
+      "rewards/chosen": 0.03295857197529561,
+      "rewards/margins": 0.013318508844717278,
+      "rewards/rejected": 0.01964006313057833,
+      "stability/repetition_rate_mean": 0.23201069235801697,
+      "stability/response_length_mean": 38.0,
+      "stability/response_length_std": 30.10457992553711,
+      "stability/response_length_var": 906.2857055664062,
+      "stability/token_entropy_mean": 4.142578125,
+      "step": 100
+    },
+    {
+      "epoch": 0.5374280230326296,
+      "grad_norm": 0.6909270882606506,
+      "kl": 0.22991745173931122,
+      "learning_rate": 9.999815554351224e-07,
+      "logits/chosen": -59385825.15662651,
+      "logits/rejected": -53806006.85714286,
+      "logps/chosen": -399.5112010542169,
+      "logps/rejected": -209.8253601866883,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4973313510417938,
+      "metrics/advantage_var": 0.24426975846290588,
+      "regularization/mmd": 0.0032832843717187643,
+      "regularization/rkhs_norm": 3.2832839488983154,
+      "rewards/chosen": 0.039128665464470185,
+      "rewards/margins": 0.02128237220084904,
+      "rewards/rejected": 0.017846293263621145,
+      "stability/repetition_rate_mean": 0.1858258843421936,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 29.89505386352539,
+      "stability/response_length_var": 893.7142944335938,
+      "stability/token_entropy_mean": 3.376953125,
+      "step": 105
+    },
+    {
+      "epoch": 0.5630198336532309,
+      "grad_norm": 0.6516240835189819,
+      "kl": 0.2624301314353943,
+      "learning_rate": 9.999066267225444e-07,
+      "logits/chosen": -57388278.746987954,
+      "logits/rejected": -53613072.62337662,
+      "logps/chosen": -378.2180440512048,
+      "logps/rejected": -205.9150897930195,
+      "loss": 0.5005,
+      "loss/base_kto": 0.4977138042449951,
+      "metrics/advantage_var": 0.21244536340236664,
+      "regularization/mmd": 0.003176226047798991,
+      "regularization/rkhs_norm": 3.1762256622314453,
+      "rewards/chosen": 0.034520548510264204,
+      "rewards/margins": 0.017216446962626662,
+      "rewards/rejected": 0.017304101547637542,
+      "stability/repetition_rate_mean": 0.30462831258773804,
+      "stability/response_length_mean": 36.375,
+      "stability/response_length_std": 30.302934646606445,
+      "stability/response_length_var": 918.2678833007812,
+      "stability/token_entropy_mean": 4.107421875,
+      "step": 110
+    },
+    {
+      "epoch": 0.5886116442738324,
+      "grad_norm": 0.6488623023033142,
+      "kl": 0.3131505250930786,
+      "learning_rate": 9.997740697079592e-07,
+      "logits/chosen": -60608589.7721519,
+      "logits/rejected": -53402333.2345679,
+      "logps/chosen": -364.07535601265823,
+      "logps/rejected": -200.99171730324073,
+      "loss": 0.5002,
+      "loss/base_kto": 0.4960899353027344,
+      "metrics/advantage_var": 0.2569291293621063,
+      "regularization/mmd": 0.0034977674949914217,
+      "regularization/rkhs_norm": 3.497767210006714,
+      "rewards/chosen": 0.05171521102325826,
+      "rewards/margins": 0.031231172682065707,
+      "rewards/rejected": 0.020484038341192553,
+      "stability/repetition_rate_mean": 0.2838541865348816,
+      "stability/response_length_mean": 41.375,
+      "stability/response_length_std": 31.25442886352539,
+      "stability/response_length_var": 976.8392944335938,
+      "stability/token_entropy_mean": 4.931640625,
+      "step": 115
+    },
+    {
+      "epoch": 0.6142034548944337,
+      "grad_norm": 0.6185635924339294,
+      "kl": 0.35157909989356995,
+      "learning_rate": 9.995838996722914e-07,
+      "logits/chosen": -57739310.849673204,
+      "logits/rejected": -54354569.964071855,
+      "logps/chosen": -383.664164624183,
+      "logps/rejected": -211.9456399700599,
+      "loss": 0.4999,
+      "loss/base_kto": 0.4961109757423401,
+      "metrics/advantage_var": 0.4056810438632965,
+      "regularization/mmd": 0.004109646659344435,
+      "regularization/rkhs_norm": 4.109646320343018,
+      "rewards/chosen": 0.05807405982921326,
+      "rewards/margins": 0.03100275802059904,
+      "rewards/rejected": 0.027071301808614216,
+      "stability/repetition_rate_mean": 0.16154514253139496,
+      "stability/response_length_mean": 27.75,
+      "stability/response_length_std": 30.358572006225586,
+      "stability/response_length_var": 921.6428833007812,
+      "stability/token_entropy_mean": 5.427734375,
+      "step": 120
+    },
+    {
+      "epoch": 0.6397952655150352,
+      "grad_norm": 0.612278938293457,
+      "kl": 0.40501686930656433,
+      "learning_rate": 9.993361385379874e-07,
+      "logits/chosen": -60694717.36986301,
+      "logits/rejected": -55665263.816091955,
+      "logps/chosen": -416.791069135274,
+      "logps/rejected": -216.09536637931035,
+      "loss": 0.5007,
+      "loss/base_kto": 0.4961540400981903,
+      "metrics/advantage_var": 0.587132453918457,
+      "regularization/mmd": 0.0047620246186852455,
+      "regularization/rkhs_norm": 4.76202392578125,
+      "rewards/chosen": 0.05882363776638083,
+      "rewards/margins": 0.03164014889863209,
+      "rewards/rejected": 0.027183488867748743,
+      "stability/repetition_rate_mean": 0.15937499701976776,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 24.93276596069336,
+      "stability/response_length_var": 621.6428833007812,
+      "stability/token_entropy_mean": 3.6484375,
+      "step": 125
+    },
+    {
+      "epoch": 0.6653870761356366,
+      "grad_norm": 0.5930764675140381,
+      "kl": 0.44866982102394104,
+      "learning_rate": 9.99030814866488e-07,
+      "logits/chosen": -57384603.2516129,
+      "logits/rejected": -53822941.86666667,
+      "logps/chosen": -391.99042338709677,
+      "logps/rejected": -183.65854640151514,
+      "loss": 0.5002,
+      "loss/base_kto": 0.49488160014152527,
+      "metrics/advantage_var": 0.7472460865974426,
+      "regularization/mmd": 0.005418591201305389,
+      "regularization/rkhs_norm": 5.418591022491455,
+      "rewards/chosen": 0.06675009573659589,
+      "rewards/margins": 0.04071308184462442,
+      "rewards/rejected": 0.026037013891971474,
+      "stability/repetition_rate_mean": 0.20924006402492523,
+      "stability/response_length_mean": 35.0,
+      "stability/response_length_std": 27.250164031982422,
+      "stability/response_length_var": 742.5714111328125,
+      "stability/token_entropy_mean": 4.01171875,
+      "step": 130
+    },
+    {
+      "epoch": 0.690978886756238,
+      "grad_norm": 0.5903881192207336,
+      "kl": 0.5167887806892395,
+      "learning_rate": 9.986679638549358e-07,
+      "logits/chosen": -59839180.17177914,
+      "logits/rejected": -54987508.58598726,
+      "logps/chosen": -400.97124233128835,
+      "logps/rejected": -247.50895700636943,
+      "loss": 0.4997,
+      "loss/base_kto": 0.4938087463378906,
+      "metrics/advantage_var": 0.9809302687644958,
+      "regularization/mmd": 0.006465181242674589,
+      "regularization/rkhs_norm": 6.465180397033691,
+      "rewards/chosen": 0.08449279457513541,
+      "rewards/margins": 0.04831445475665474,
+      "rewards/rejected": 0.03617833981848067,
+      "stability/repetition_rate_mean": 0.2230902761220932,
+      "stability/response_length_mean": 29.25,
+      "stability/response_length_std": 29.936838150024414,
+      "stability/response_length_var": 896.2142944335938,
+      "stability/token_entropy_mean": 4.078125,
+      "step": 135
+    },
+    {
+      "epoch": 0.7165706973768394,
+      "grad_norm": 0.7281087636947632,
+      "kl": 0.48667556047439575,
+      "learning_rate": 9.982476273321175e-07,
+      "logits/chosen": -57080863.03030303,
+      "logits/rejected": -50078974.3483871,
+      "logps/chosen": -366.377959280303,
+      "logps/rejected": -189.92201360887097,
+      "loss": 0.4993,
+      "loss/base_kto": 0.4917573034763336,
+      "metrics/advantage_var": 1.9869550466537476,
+      "regularization/mmd": 0.0070263417437672615,
+      "regularization/rkhs_norm": 7.026341438293457,
+      "rewards/chosen": 0.08117941654089725,
+      "rewards/margins": 0.06993257689452707,
+      "rewards/rejected": 0.01124683964637018,
+      "stability/repetition_rate_mean": 0.2858072817325592,
+      "stability/response_length_mean": 52.75,
+      "stability/response_length_std": 22.72663688659668,
+      "stability/response_length_var": 516.5,
+      "stability/token_entropy_mean": 5.0244140625,
+      "step": 140
+    },
+    {
+      "epoch": 0.7421625079974408,
+      "grad_norm": 0.6801453232765198,
+      "kl": 0.564969003200531,
+      "learning_rate": 9.977698537536417e-07,
+      "logits/chosen": -51148756.81927711,
+      "logits/rejected": -51663512.935064934,
+      "logps/chosen": -329.46656155873495,
+      "logps/rejected": -199.1533710430195,
+      "loss": 0.4994,
+      "loss/base_kto": 0.4940341114997864,
+      "metrics/advantage_var": 0.8375579714775085,
+      "regularization/mmd": 0.006148753222078085,
+      "regularization/rkhs_norm": 6.148753643035889,
+      "rewards/chosen": 0.08052689196115517,
+      "rewards/margins": 0.0470729196524885,
+      "rewards/rejected": 0.033453972308666674,
+      "stability/repetition_rate_mean": 0.18058820068836212,
+      "stability/response_length_mean": 22.625,
+      "stability/response_length_std": 26.359804153442383,
+      "stability/response_length_var": 694.8392944335938,
+      "stability/token_entropy_mean": 5.72265625,
+      "step": 145
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.6427031755447388,
+      "kl": 0.6379441022872925,
+      "learning_rate": 9.972346981963546e-07,
+      "logits/chosen": -55423662.68235294,
+      "logits/rejected": -50186574.50666667,
+      "logps/chosen": -396.80716911764705,
+      "logps/rejected": -207.3796875,
+      "loss": 0.4998,
+      "loss/base_kto": 0.4897971749305725,
+      "metrics/advantage_var": 1.2700830698013306,
+      "regularization/mmd": 0.007216709200292826,
+      "regularization/rkhs_norm": 7.216709136962891,
+      "rewards/chosen": 0.11272701936609605,
+      "rewards/margins": 0.07810217576868395,
+      "rewards/rejected": 0.03462484359741211,
+      "stability/repetition_rate_mean": 0.2408854216337204,
+      "stability/response_length_mean": 35.25,
+      "stability/response_length_std": 31.01957893371582,
+      "stability/response_length_var": 962.2142944335938,
+      "stability/token_entropy_mean": 4.64453125,
+      "step": 150
+    },
+    {
+      "epoch": 0.7933461292386437,
+      "grad_norm": 0.6279340982437134,
+      "kl": 0.6466606855392456,
+      "learning_rate": 9.966422223519885e-07,
+      "logits/chosen": -54007579.105882354,
+      "logits/rejected": -49333650.77333333,
+      "logps/chosen": -360.1019071691176,
+      "logps/rejected": -191.019296875,
+      "loss": 0.5004,
+      "loss/base_kto": 0.4928290843963623,
+      "metrics/advantage_var": 2.206279754638672,
+      "regularization/mmd": 0.008163990452885628,
+      "regularization/rkhs_norm": 8.163989067077637,
+      "rewards/chosen": 0.08873968124389649,
+      "rewards/margins": 0.060108397801717124,
+      "rewards/rejected": 0.028631283442179363,
+      "stability/repetition_rate_mean": 0.3057386875152588,
+      "stability/response_length_mean": 54.25,
+      "stability/response_length_std": 20.457273483276367,
+      "stability/response_length_var": 418.5,
+      "stability/token_entropy_mean": 4.7119140625,
+      "step": 155
+    },
+    {
+      "epoch": 0.818937939859245,
+      "grad_norm": 0.6161509156227112,
+      "kl": 0.7146582007408142,
+      "learning_rate": 9.959924945200523e-07,
+      "logits/chosen": -55253735.61904762,
+      "logits/rejected": -48960865.684210524,
+      "logps/chosen": -415.5394810267857,
+      "logps/rejected": -186.50149054276315,
+      "loss": 0.4999,
+      "loss/base_kto": 0.4877723157405853,
+      "metrics/advantage_var": 1.9780712127685547,
+      "regularization/mmd": 0.00858414825052023,
+      "regularization/rkhs_norm": 8.584148406982422,
+      "rewards/chosen": 0.13558682941255115,
+      "rewards/margins": 0.09600058967308292,
+      "rewards/rejected": 0.039586239739468225,
+      "stability/repetition_rate_mean": 0.30851560831069946,
+      "stability/response_length_mean": 32.5,
+      "stability/response_length_std": 30.528675079345703,
+      "stability/response_length_var": 932.0,
+      "stability/token_entropy_mean": 4.44921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.8445297504798465,
+      "grad_norm": 0.7212228775024414,
+      "kl": 0.7538337707519531,
+      "learning_rate": 9.952855895999568e-07,
+      "logits/chosen": -54409180.68965517,
+      "logits/rejected": -49792936.32876712,
+      "logps/chosen": -369.73284841954023,
+      "logps/rejected": -212.12114726027397,
+      "loss": 0.499,
+      "loss/base_kto": 0.4910934567451477,
+      "metrics/advantage_var": 1.8075367212295532,
+      "regularization/mmd": 0.008565567433834076,
+      "regularization/rkhs_norm": 8.565567016601562,
+      "rewards/chosen": 0.11838912963867188,
+      "rewards/margins": 0.07129266164074205,
+      "rewards/rejected": 0.04709646799792982,
+      "stability/repetition_rate_mean": 0.2411106526851654,
+      "stability/response_length_mean": 22.5,
+      "stability/response_length_std": 28.33977508544922,
+      "stability/response_length_var": 803.1428833007812,
+      "stability/token_entropy_mean": 4.85546875,
+      "step": 165
+    },
+    {
+      "epoch": 0.8701215611004478,
+      "grad_norm": 0.6235238313674927,
+      "kl": 0.9214170575141907,
+      "learning_rate": 9.9452158908238e-07,
+      "logits/chosen": -59134242.840236686,
+      "logits/rejected": -52560461.98675497,
+      "logps/chosen": -439.6619822485207,
+      "logps/rejected": -190.60081229304635,
+      "loss": 0.4981,
+      "loss/base_kto": 0.4872615933418274,
+      "metrics/advantage_var": 2.389000654220581,
+      "regularization/mmd": 0.010343970730900764,
+      "regularization/rkhs_norm": 10.34397029876709,
+      "rewards/chosen": 0.16798810563849273,
+      "rewards/margins": 0.1022866525369367,
+      "rewards/rejected": 0.06570145310155603,
+      "stability/repetition_rate_mean": 0.3512404263019562,
+      "stability/response_length_mean": 44.125,
+      "stability/response_length_std": 28.43256378173828,
+      "stability/response_length_var": 808.4107055664062,
+      "stability/token_entropy_mean": 3.58251953125,
+      "step": 170
+    },
+    {
+      "epoch": 0.8957133717210493,
+      "grad_norm": 0.7214075922966003,
+      "kl": 0.9537512063980103,
+      "learning_rate": 9.937005810398745e-07,
+      "logits/chosen": -56639210.409638554,
+      "logits/rejected": -48014429.09090909,
+      "logps/chosen": -354.86631682981925,
+      "logps/rejected": -194.32470068993507,
+      "loss": 0.4983,
+      "loss/base_kto": 0.48649540543556213,
+      "metrics/advantage_var": 4.312074184417725,
+      "regularization/mmd": 0.012221619486808777,
+      "regularization/rkhs_norm": 12.22161865234375,
+      "rewards/chosen": 0.15708599320377212,
+      "rewards/margins": 0.10880196726838853,
+      "rewards/rejected": 0.048284025935383584,
+      "stability/repetition_rate_mean": 0.18430830538272858,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 25.773740768432617,
+      "stability/response_length_var": 664.2857055664062,
+      "stability/token_entropy_mean": 4.865234375,
+      "step": 175
+    },
+    {
+      "epoch": 0.9213051823416507,
+      "grad_norm": 0.6424931287765503,
+      "kl": 1.0256928205490112,
+      "learning_rate": 9.928226601167138e-07,
+      "logits/chosen": -52637997.54601227,
+      "logits/rejected": -47981639.74522293,
+      "logps/chosen": -374.39994727760734,
+      "logps/rejected": -210.55190087579618,
+      "loss": 0.4982,
+      "loss/base_kto": 0.4874132573604584,
+      "metrics/advantage_var": 4.2638068199157715,
+      "regularization/mmd": 0.012582081370055676,
+      "regularization/rkhs_norm": 12.582079887390137,
+      "rewards/chosen": 0.1745892858212711,
+      "rewards/margins": 0.09995884616016004,
+      "rewards/rejected": 0.07463043966111105,
+      "stability/repetition_rate_mean": 0.25468748807907104,
+      "stability/response_length_mean": 34.75,
+      "stability/response_length_std": 31.54928207397461,
+      "stability/response_length_var": 995.3571166992188,
+      "stability/token_entropy_mean": 3.251953125,
+      "step": 180
+    },
+    {
+      "epoch": 0.946896992962252,
+      "grad_norm": 0.6472417116165161,
+      "kl": 1.0763963460922241,
+      "learning_rate": 9.918879275179817e-07,
+      "logits/chosen": -54603801.28395062,
+      "logits/rejected": -49515623.69620253,
+      "logps/chosen": -310.3162856867284,
+      "logps/rejected": -196.56774129746836,
+      "loss": 0.4988,
+      "loss/base_kto": 0.48661041259765625,
+      "metrics/advantage_var": 7.496189117431641,
+      "regularization/mmd": 0.014902687631547451,
+      "regularization/rkhs_norm": 14.90268611907959,
+      "rewards/chosen": 0.16850916544596353,
+      "rewards/margins": 0.1213674957742168,
+      "rewards/rejected": 0.047141669671746746,
+      "stability/repetition_rate_mean": 0.197265625,
+      "stability/response_length_mean": 32.875,
+      "stability/response_length_std": 33.30996322631836,
+      "stability/response_length_var": 1109.5535888671875,
+      "stability/token_entropy_mean": 3.66015625,
+      "step": 185
+    },
+    {
+      "epoch": 0.9724888035828535,
+      "grad_norm": 0.6970455646514893,
+      "kl": 1.1167560815811157,
+      "learning_rate": 9.90896490997906e-07,
+      "logits/chosen": -53093547.68862276,
+      "logits/rejected": -49684118.5882353,
+      "logps/chosen": -390.3140671781437,
+      "logps/rejected": -210.4685968137255,
+      "loss": 0.4993,
+      "loss/base_kto": 0.48344355821609497,
+      "metrics/advantage_var": 7.254169464111328,
+      "regularization/mmd": 0.016163000836968422,
+      "regularization/rkhs_norm": 16.163000106811523,
+      "rewards/chosen": 0.209123828453932,
+      "rewards/margins": 0.13612457234483144,
+      "rewards/rejected": 0.07299925610910055,
+      "stability/repetition_rate_mean": 0.1931029111146927,
+      "stability/response_length_mean": 14.25,
+      "stability/response_length_std": 21.743637084960938,
+      "stability/response_length_var": 472.78570556640625,
+      "stability/token_entropy_mean": 4.82421875,
+      "step": 190
+    },
+    {
+      "epoch": 0.9980806142034548,
+      "grad_norm": 0.6390827298164368,
+      "kl": 1.1463875770568848,
+      "learning_rate": 9.898484648474362e-07,
+      "logits/chosen": -60031385.6,
+      "logits/rejected": -53806036.557575755,
+      "logps/chosen": -390.87472278225806,
+      "logps/rejected": -197.0807646780303,
+      "loss": 0.4996,
+      "loss/base_kto": 0.48643895983695984,
+      "metrics/advantage_var": 4.362728118896484,
+      "regularization/mmd": 0.013747951947152615,
+      "regularization/rkhs_norm": 13.747952461242676,
+      "rewards/chosen": 0.1910219254032258,
+      "rewards/margins": 0.1071656144143083,
+      "rewards/rejected": 0.08385631098891749,
+      "stability/repetition_rate_mean": 0.3934151828289032,
+      "stability/response_length_mean": 49.0,
+      "stability/response_length_std": 27.82085609436035,
+      "stability/response_length_var": 774.0,
+      "stability/token_entropy_mean": 5.18359375,
+      "step": 195
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 196,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.1861866257511875e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da67b5a16dad8f0b2557e1d67989a1e387521af245b52b543853b881a35dbad4
+size 8081

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/checkpoint-196/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/stability_metrics.jsonl

@@ -0,0 +1,38 @@
+{"stability/response_length_mean": 46.125, "stability/response_length_std": 27.231481552124023, "stability/response_length_var": 741.5535888671875, "stability/token_entropy_mean": 3.29638671875, "stability/repetition_rate_mean": 0.3570815920829773, "step": 10}
+{"stability/response_length_mean": 27.125, "stability/response_length_std": 30.79163932800293, "stability/response_length_var": 948.125, "stability/token_entropy_mean": 3.82421875, "stability/repetition_rate_mean": 0.2651909589767456, "step": 15}
+{"stability/response_length_mean": 44.0, "stability/response_length_std": 28.0611572265625, "stability/response_length_var": 787.4285888671875, "stability/token_entropy_mean": 4.42578125, "stability/repetition_rate_mean": 0.2155257910490036, "step": 20}
+{"stability/response_length_mean": 43.375, "stability/response_length_std": 28.54538917541504, "stability/response_length_var": 814.8392944335938, "stability/token_entropy_mean": 4.763671875, "stability/repetition_rate_mean": 0.17606504261493683, "step": 25}
+{"stability/response_length_mean": 43.0, "stability/response_length_std": 29.602123260498047, "stability/response_length_var": 876.2857055664062, "stability/token_entropy_mean": 4.552734375, "stability/repetition_rate_mean": 0.1544828861951828, "step": 30}
+{"stability/response_length_mean": 53.375, "stability/response_length_std": 21.14533042907715, "stability/response_length_var": 447.125, "stability/token_entropy_mean": 5.326171875, "stability/repetition_rate_mean": 0.34880331158638, "step": 35}
+{"stability/response_length_mean": 33.125, "stability/response_length_std": 33.01704025268555, "stability/response_length_var": 1090.125, "stability/token_entropy_mean": 4.9765625, "stability/repetition_rate_mean": 0.142578125, "step": 40}
+{"stability/response_length_mean": 29.25, "stability/response_length_std": 29.51875877380371, "stability/response_length_var": 871.3571166992188, "stability/token_entropy_mean": 4.708984375, "stability/repetition_rate_mean": 0.1013549193739891, "step": 45}
+{"stability/response_length_mean": 55.375, "stability/response_length_std": 16.361431121826172, "stability/response_length_var": 267.6964416503906, "stability/token_entropy_mean": 3.814453125, "stability/repetition_rate_mean": 0.4108419120311737, "step": 50}
+{"stability/response_length_mean": 22.375, "stability/response_length_std": 30.236862182617188, "stability/response_length_var": 914.2678833007812, "stability/token_entropy_mean": 4.5703125, "stability/repetition_rate_mean": 0.10613363981246948, "step": 55}
+{"stability/response_length_mean": 51.125, "stability/response_length_std": 21.90523338317871, "stability/response_length_var": 479.83929443359375, "stability/token_entropy_mean": 5.353515625, "stability/repetition_rate_mean": 0.20835673809051514, "step": 60}
+{"stability/response_length_mean": 17.875, "stability/response_length_std": 23.485177993774414, "stability/response_length_var": 551.5535888671875, "stability/token_entropy_mean": 5.66796875, "stability/repetition_rate_mean": 0.18097305297851562, "step": 65}
+{"stability/response_length_mean": 33.125, "stability/response_length_std": 33.0300178527832, "stability/response_length_var": 1090.982177734375, "stability/token_entropy_mean": 5.765625, "stability/repetition_rate_mean": 0.1875, "step": 70}
+{"stability/response_length_mean": 22.75, "stability/response_length_std": 31.07479476928711, "stability/response_length_var": 965.6428833007812, "stability/token_entropy_mean": 2.16015625, "stability/repetition_rate_mean": 0.06355167925357819, "step": 75}
+{"stability/response_length_mean": 42.0, "stability/response_length_std": 30.481843948364258, "stability/response_length_var": 929.1428833007812, "stability/token_entropy_mean": 5.25, "stability/repetition_rate_mean": 0.18697917461395264, "step": 80}
+{"stability/response_length_mean": 27.375, "stability/response_length_std": 30.39707374572754, "stability/response_length_var": 923.9821166992188, "stability/token_entropy_mean": 4.91796875, "stability/repetition_rate_mean": 0.1269342005252838, "step": 85}
+{"stability/response_length_mean": 32.875, "stability/response_length_std": 30.31000518798828, "stability/response_length_var": 918.6964111328125, "stability/token_entropy_mean": 3.001953125, "stability/repetition_rate_mean": 0.2901870310306549, "step": 90}
+{"stability/response_length_mean": 27.0, "stability/response_length_std": 31.1264705657959, "stability/response_length_var": 968.8571166992188, "stability/token_entropy_mean": 3.689453125, "stability/repetition_rate_mean": 0.19764858484268188, "step": 95}
+{"stability/response_length_mean": 38.0, "stability/response_length_std": 30.10457992553711, "stability/response_length_var": 906.2857055664062, "stability/token_entropy_mean": 4.142578125, "stability/repetition_rate_mean": 0.23201069235801697, "step": 100}
+{"stability/response_length_mean": 29.0, "stability/response_length_std": 29.89505386352539, "stability/response_length_var": 893.7142944335938, "stability/token_entropy_mean": 3.376953125, "stability/repetition_rate_mean": 0.1858258843421936, "step": 105}
+{"stability/response_length_mean": 36.375, "stability/response_length_std": 30.302934646606445, "stability/response_length_var": 918.2678833007812, "stability/token_entropy_mean": 4.107421875, "stability/repetition_rate_mean": 0.30462831258773804, "step": 110}
+{"stability/response_length_mean": 41.375, "stability/response_length_std": 31.25442886352539, "stability/response_length_var": 976.8392944335938, "stability/token_entropy_mean": 4.931640625, "stability/repetition_rate_mean": 0.2838541865348816, "step": 115}
+{"stability/response_length_mean": 27.75, "stability/response_length_std": 30.358572006225586, "stability/response_length_var": 921.6428833007812, "stability/token_entropy_mean": 5.427734375, "stability/repetition_rate_mean": 0.16154514253139496, "step": 120}
+{"stability/response_length_mean": 15.25, "stability/response_length_std": 24.93276596069336, "stability/response_length_var": 621.6428833007812, "stability/token_entropy_mean": 3.6484375, "stability/repetition_rate_mean": 0.15937499701976776, "step": 125}
+{"stability/response_length_mean": 35.0, "stability/response_length_std": 27.250164031982422, "stability/response_length_var": 742.5714111328125, "stability/token_entropy_mean": 4.01171875, "stability/repetition_rate_mean": 0.20924006402492523, "step": 130}
+{"stability/response_length_mean": 29.25, "stability/response_length_std": 29.936838150024414, "stability/response_length_var": 896.2142944335938, "stability/token_entropy_mean": 4.078125, "stability/repetition_rate_mean": 0.2230902761220932, "step": 135}
+{"stability/response_length_mean": 52.75, "stability/response_length_std": 22.72663688659668, "stability/response_length_var": 516.5, "stability/token_entropy_mean": 5.0244140625, "stability/repetition_rate_mean": 0.2858072817325592, "step": 140}
+{"stability/response_length_mean": 22.625, "stability/response_length_std": 26.359804153442383, "stability/response_length_var": 694.8392944335938, "stability/token_entropy_mean": 5.72265625, "stability/repetition_rate_mean": 0.18058820068836212, "step": 145}
+{"stability/response_length_mean": 35.25, "stability/response_length_std": 31.01957893371582, "stability/response_length_var": 962.2142944335938, "stability/token_entropy_mean": 4.64453125, "stability/repetition_rate_mean": 0.2408854216337204, "step": 150}
+{"stability/response_length_mean": 54.25, "stability/response_length_std": 20.457273483276367, "stability/response_length_var": 418.5, "stability/token_entropy_mean": 4.7119140625, "stability/repetition_rate_mean": 0.3057386875152588, "step": 155}
+{"stability/response_length_mean": 32.5, "stability/response_length_std": 30.528675079345703, "stability/response_length_var": 932.0, "stability/token_entropy_mean": 4.44921875, "stability/repetition_rate_mean": 0.30851560831069946, "step": 160}
+{"stability/response_length_mean": 22.5, "stability/response_length_std": 28.33977508544922, "stability/response_length_var": 803.1428833007812, "stability/token_entropy_mean": 4.85546875, "stability/repetition_rate_mean": 0.2411106526851654, "step": 165}
+{"stability/response_length_mean": 44.125, "stability/response_length_std": 28.43256378173828, "stability/response_length_var": 808.4107055664062, "stability/token_entropy_mean": 3.58251953125, "stability/repetition_rate_mean": 0.3512404263019562, "step": 170}
+{"stability/response_length_mean": 27.0, "stability/response_length_std": 25.773740768432617, "stability/response_length_var": 664.2857055664062, "stability/token_entropy_mean": 4.865234375, "stability/repetition_rate_mean": 0.18430830538272858, "step": 175}
+{"stability/response_length_mean": 34.75, "stability/response_length_std": 31.54928207397461, "stability/response_length_var": 995.3571166992188, "stability/token_entropy_mean": 3.251953125, "stability/repetition_rate_mean": 0.25468748807907104, "step": 180}
+{"stability/response_length_mean": 32.875, "stability/response_length_std": 33.30996322631836, "stability/response_length_var": 1109.5535888671875, "stability/token_entropy_mean": 3.66015625, "stability/repetition_rate_mean": 0.197265625, "step": 185}
+{"stability/response_length_mean": 14.25, "stability/response_length_std": 21.743637084960938, "stability/response_length_var": 472.78570556640625, "stability/token_entropy_mean": 4.82421875, "stability/repetition_rate_mean": 0.1931029111146927, "step": 190}
+{"stability/response_length_mean": 49.0, "stability/response_length_std": 27.82085609436035, "stability/response_length_var": 774.0, "stability/token_entropy_mean": 5.18359375, "stability/repetition_rate_mean": 0.3934151828289032, "step": 195}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.1861866257511875e+18,
+    "train_loss": 0.500500318955402,
+    "train_runtime": 3156.0327,
+    "train_samples_per_second": 31.685,
+    "train_steps_per_second": 0.062
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/trainer_log.jsonl

@@ -0,0 +1,40 @@
+{"current_steps": 5, "total_steps": 196, "loss": 0.5009, "lr": 4e-08, "epoch": 0.025591810620601407, "percentage": 2.55, "elapsed_time": "0:01:19", "remaining_time": "0:50:23", "rewards/chosen": 0.005160976397363763, "logps/chosen": -438.99403782894734, "logits/chosen": -59712215.578947365, "rewards/rejected": 0.00030076858543214344, "logps/rejected": -198.40638950892858, "logits/rejected": -50420742.0952381, "rewards/margins": 0.004860207811931619, "regularization/mmd": 0.0012165767839178443, "regularization/rkhs_norm": 1.2165768146514893, "kl": 0.008188384585082531}
+{"current_steps": 10, "total_steps": 196, "loss": 0.5018, "lr": 9e-08, "epoch": 0.05118362124120281, "percentage": 5.1, "elapsed_time": "0:02:40", "remaining_time": "0:49:37", "rewards/chosen": 0.001513990101876197, "logps/chosen": -429.9233969155844, "logits/chosen": -59291993.766233765, "rewards/rejected": -0.0010130622659821108, "logps/rejected": -192.87807087725903, "logits/rejected": -50017332.43373494, "rewards/margins": 0.0025270523678583077, "regularization/mmd": 0.0020083182025700808, "regularization/rkhs_norm": 2.0083181858062744, "kl": 0.017390631139278412, "stability/response_length_mean": 46.125, "stability/response_length_std": 27.231481552124023, "stability/response_length_var": 741.5535888671875, "stability/token_entropy_mean": 3.29638671875, "stability/repetition_rate_mean": 0.3570815920829773}
+{"current_steps": 15, "total_steps": 196, "loss": 0.5016, "lr": 1.4e-07, "epoch": 0.07677543186180422, "percentage": 7.65, "elapsed_time": "0:04:00", "remaining_time": "0:48:16", "rewards/chosen": 0.0011517706242474644, "logps/chosen": -404.1193181818182, "logits/chosen": -56235380.36363637, "rewards/rejected": 0.0015282316553977228, "logps/rejected": -200.67879284274193, "logits/rejected": -51927694.03870968, "rewards/margins": -0.0003764610311502584, "regularization/mmd": 0.0016775511903688312, "regularization/rkhs_norm": 1.6775509119033813, "kl": 0.00862761028110981, "stability/response_length_mean": 27.125, "stability/response_length_std": 30.79163932800293, "stability/response_length_var": 948.125, "stability/token_entropy_mean": 3.82421875, "stability/repetition_rate_mean": 0.2651909589767456}
+{"current_steps": 20, "total_steps": 196, "loss": 0.5019, "lr": 1.8999999999999998e-07, "epoch": 0.10236724248240563, "percentage": 10.2, "elapsed_time": "0:05:21", "remaining_time": "0:47:09", "rewards/chosen": 0.002454766258597374, "logps/chosen": -343.67216796875, "logits/chosen": -51664051.2, "rewards/rejected": 0.0023710768669843673, "logps/rejected": -225.6396484375, "logits/rejected": -48054336.0, "rewards/margins": 8.368939161300685e-05, "regularization/mmd": 0.001691564917564392, "regularization/rkhs_norm": 1.691564917564392, "kl": 0.011922934092581272, "stability/response_length_mean": 44.0, "stability/response_length_std": 28.0611572265625, "stability/response_length_var": 787.4285888671875, "stability/token_entropy_mean": 4.42578125, "stability/repetition_rate_mean": 0.2155257910490036}
+{"current_steps": 25, "total_steps": 196, "loss": 0.5019, "lr": 2.4e-07, "epoch": 0.12795905310300704, "percentage": 12.76, "elapsed_time": "0:06:41", "remaining_time": "0:45:46", "rewards/chosen": 0.0026301628556744806, "logps/chosen": -378.4651400862069, "logits/chosen": -56476908.57931034, "rewards/rejected": -0.00025400987693241663, "logps/rejected": -202.7953794642857, "logits/rejected": -52024659.382857144, "rewards/margins": 0.0028841727326068974, "regularization/mmd": 0.0017472734907642007, "regularization/rkhs_norm": 1.7472734451293945, "kl": 0.018707742914557457, "stability/response_length_mean": 43.375, "stability/response_length_std": 28.54538917541504, "stability/response_length_var": 814.8392944335938, "stability/token_entropy_mean": 4.763671875, "stability/repetition_rate_mean": 0.17606504261493683}
+{"current_steps": 30, "total_steps": 196, "loss": 0.5019, "lr": 2.9e-07, "epoch": 0.15355086372360843, "percentage": 15.31, "elapsed_time": "0:08:01", "remaining_time": "0:44:25", "rewards/chosen": 0.0012253856898954251, "logps/chosen": -468.5690016778523, "logits/chosen": -57730419.11409396, "rewards/rejected": 0.005818910766066166, "logps/rejected": -198.18969298245614, "logits/rejected": -51521314.43274854, "rewards/margins": -0.004593525076170741, "regularization/mmd": 0.0019578204955905676, "regularization/rkhs_norm": 1.9578202962875366, "kl": 0.017847830429673195, "stability/response_length_mean": 43.0, "stability/response_length_std": 29.602123260498047, "stability/response_length_var": 876.2857055664062, "stability/token_entropy_mean": 4.552734375, "stability/repetition_rate_mean": 0.1544828861951828}
+{"current_steps": 35, "total_steps": 196, "loss": 0.5017, "lr": 3.4000000000000003e-07, "epoch": 0.17914267434420986, "percentage": 17.86, "elapsed_time": "0:09:23", "remaining_time": "0:43:10", "rewards/chosen": 0.0029728393765944467, "logps/chosen": -385.90676918512656, "logits/chosen": -57633720.70886076, "rewards/rejected": -0.0004626273373026907, "logps/rejected": -195.13110050154322, "logits/rejected": -53879119.01234568, "rewards/margins": 0.0034354667138971375, "regularization/mmd": 0.0018217876786366105, "regularization/rkhs_norm": 1.821787714958191, "kl": 0.021474814042448997, "stability/response_length_mean": 53.375, "stability/response_length_std": 21.14533042907715, "stability/response_length_var": 447.125, "stability/token_entropy_mean": 5.326171875, "stability/repetition_rate_mean": 0.34880331158638}
+{"current_steps": 40, "total_steps": 196, "loss": 0.5019, "lr": 3.8999999999999997e-07, "epoch": 0.20473448496481125, "percentage": 20.41, "elapsed_time": "0:10:43", "remaining_time": "0:41:51", "rewards/chosen": -0.003416117653250694, "logps/chosen": -421.93681640625, "logits/chosen": -56637171.2, "rewards/rejected": 0.006124629080295563, "logps/rejected": -219.247998046875, "logits/rejected": -50217504.0, "rewards/margins": -0.009540746733546258, "regularization/mmd": 0.0019648734014481306, "regularization/rkhs_norm": 1.9648735523223877, "kl": 0.018473921343684196, "stability/response_length_mean": 33.125, "stability/response_length_std": 33.01704025268555, "stability/response_length_var": 1090.125, "stability/token_entropy_mean": 4.9765625, "stability/repetition_rate_mean": 0.142578125}
+{"current_steps": 45, "total_steps": 196, "loss": 0.5017, "lr": 4.3999999999999997e-07, "epoch": 0.23032629558541268, "percentage": 22.96, "elapsed_time": "0:12:04", "remaining_time": "0:40:31", "rewards/chosen": 0.0035465471696533614, "logps/chosen": -352.29299496644296, "logits/chosen": -52695545.12751678, "rewards/rejected": 0.004557689728095517, "logps/rejected": -243.82858187134502, "logits/rejected": -50770458.94736842, "rewards/margins": -0.0010111425584421558, "regularization/mmd": 0.0017398166237398982, "regularization/rkhs_norm": 1.739816665649414, "kl": 0.032560475170612335, "stability/response_length_mean": 29.25, "stability/response_length_std": 29.51875877380371, "stability/response_length_var": 871.3571166992188, "stability/token_entropy_mean": 4.708984375, "stability/repetition_rate_mean": 0.1013549193739891}
+{"current_steps": 50, "total_steps": 196, "loss": 0.5015, "lr": 4.9e-07, "epoch": 0.2559181062060141, "percentage": 25.51, "elapsed_time": "0:13:24", "remaining_time": "0:39:09", "rewards/chosen": 0.007055705025692114, "logps/chosen": -375.1285654362416, "logits/chosen": -54038620.77852349, "rewards/rejected": 0.0033609246649937323, "logps/rejected": -180.20700840643275, "logits/rejected": -48295651.55555555, "rewards/margins": 0.0036947803606983817, "regularization/mmd": 0.0017960931872949004, "regularization/rkhs_norm": 1.7960929870605469, "kl": 0.04730672389268875, "stability/response_length_mean": 55.375, "stability/response_length_std": 16.361431121826172, "stability/response_length_var": 267.6964416503906, "stability/token_entropy_mean": 3.814453125, "stability/repetition_rate_mean": 0.4108419120311737}
+{"current_steps": 55, "total_steps": 196, "loss": 0.5015, "lr": 5.4e-07, "epoch": 0.28150991682661547, "percentage": 28.06, "elapsed_time": "0:14:44", "remaining_time": "0:37:47", "rewards/chosen": 0.004102396668854707, "logps/chosen": -344.4710064052795, "logits/chosen": -56956718.111801244, "rewards/rejected": 0.005811771506783347, "logps/rejected": -230.38991253930817, "logits/rejected": -49555314.31446541, "rewards/margins": -0.0017093748379286402, "regularization/mmd": 0.001749949180521071, "regularization/rkhs_norm": 1.7499490976333618, "kl": 0.05266108736395836, "stability/response_length_mean": 22.375, "stability/response_length_std": 30.236862182617188, "stability/response_length_var": 914.2678833007812, "stability/token_entropy_mean": 4.5703125, "stability/repetition_rate_mean": 0.10613363981246948}
+{"current_steps": 60, "total_steps": 196, "loss": 0.5015, "lr": 5.9e-07, "epoch": 0.30710172744721687, "percentage": 30.61, "elapsed_time": "0:16:06", "remaining_time": "0:36:29", "rewards/chosen": 0.007016492612434156, "logps/chosen": -359.6506628787879, "logits/chosen": -52669142.10909091, "rewards/rejected": 0.009258242576353012, "logps/rejected": -178.1343245967742, "logits/rejected": -47815765.88387097, "rewards/margins": -0.0022417499639188557, "regularization/mmd": 0.0018729189177975059, "regularization/rkhs_norm": 1.872918963432312, "kl": 0.050893235951662064, "stability/response_length_mean": 51.125, "stability/response_length_std": 21.90523338317871, "stability/response_length_var": 479.83929443359375, "stability/token_entropy_mean": 5.353515625, "stability/repetition_rate_mean": 0.20835673809051514}
+{"current_steps": 65, "total_steps": 196, "loss": 0.5013, "lr": 6.4e-07, "epoch": 0.3326935380678183, "percentage": 33.16, "elapsed_time": "0:17:26", "remaining_time": "0:35:09", "rewards/chosen": 0.012116812121483588, "logps/chosen": -381.8256048387097, "logits/chosen": -57555868.90322581, "rewards/rejected": 0.007160423741196141, "logps/rejected": -183.51074810606062, "logits/rejected": -47274523.92727273, "rewards/margins": 0.004956388380287447, "regularization/mmd": 0.0016916670138016343, "regularization/rkhs_norm": 1.6916671991348267, "kl": 0.06566640734672546, "stability/response_length_mean": 17.875, "stability/response_length_std": 23.485177993774414, "stability/response_length_var": 551.5535888671875, "stability/token_entropy_mean": 5.66796875, "stability/repetition_rate_mean": 0.18097305297851562}
+{"current_steps": 70, "total_steps": 196, "loss": 0.5012, "lr": 6.9e-07, "epoch": 0.3582853486884197, "percentage": 35.71, "elapsed_time": "0:18:46", "remaining_time": "0:33:48", "rewards/chosen": 0.01297938746790732, "logps/chosen": -378.99866431451613, "logits/chosen": -52775096.980645165, "rewards/rejected": 0.007156934160174745, "logps/rejected": -220.35059185606062, "logits/rejected": -49028474.56969697, "rewards/margins": 0.0058224533077325755, "regularization/mmd": 0.001765405759215355, "regularization/rkhs_norm": 1.7654056549072266, "kl": 0.08278707414865494, "stability/response_length_mean": 33.125, "stability/response_length_std": 33.0300178527832, "stability/response_length_var": 1090.982177734375, "stability/token_entropy_mean": 5.765625, "stability/repetition_rate_mean": 0.1875}
+{"current_steps": 75, "total_steps": 196, "loss": 0.5014, "lr": 7.4e-07, "epoch": 0.3838771593090211, "percentage": 38.27, "elapsed_time": "0:20:07", "remaining_time": "0:32:28", "rewards/chosen": 0.012649115678426382, "logps/chosen": -314.14432010135135, "logits/chosen": -49496012.10810811, "rewards/rejected": 0.007269135741300361, "logps/rejected": -228.81213662790697, "logits/rejected": -47883255.069767445, "rewards/margins": 0.005379979937126021, "regularization/mmd": 0.0018627401441335678, "regularization/rkhs_norm": 1.862740397453308, "kl": 0.10445437580347061, "stability/response_length_mean": 22.75, "stability/response_length_std": 31.07479476928711, "stability/response_length_var": 965.6428833007812, "stability/token_entropy_mean": 2.16015625, "stability/repetition_rate_mean": 0.06355167925357819}
+{"current_steps": 80, "total_steps": 196, "loss": 0.5012, "lr": 7.9e-07, "epoch": 0.4094689699296225, "percentage": 40.82, "elapsed_time": "0:21:29", "remaining_time": "0:31:09", "rewards/chosen": 0.01823797131216289, "logps/chosen": -403.8477597268212, "logits/chosen": -58612227.39072847, "rewards/rejected": 0.007570893101438263, "logps/rejected": -200.98007581360946, "logits/rejected": -53431647.43195266, "rewards/margins": 0.010667078210724629, "regularization/mmd": 0.0019349249778315425, "regularization/rkhs_norm": 1.9349247217178345, "kl": 0.11964090168476105, "stability/response_length_mean": 42.0, "stability/response_length_std": 30.481843948364258, "stability/response_length_var": 929.1428833007812, "stability/token_entropy_mean": 5.25, "stability/repetition_rate_mean": 0.18697917461395264}
+{"current_steps": 85, "total_steps": 196, "loss": 0.5007, "lr": 8.399999999999999e-07, "epoch": 0.4350607805502239, "percentage": 43.37, "elapsed_time": "0:22:49", "remaining_time": "0:29:48", "rewards/chosen": 0.021053169619652534, "logps/chosen": -358.1219758064516, "logits/chosen": -51474676.438709676, "rewards/rejected": 0.006881673408277107, "logps/rejected": -230.35598958333333, "logits/rejected": -47137016.24242424, "rewards/margins": 0.014171496211375426, "regularization/mmd": 0.0020756726153194904, "regularization/rkhs_norm": 2.0756723880767822, "kl": 0.12551236152648926, "stability/response_length_mean": 27.375, "stability/response_length_std": 30.39707374572754, "stability/response_length_var": 923.9821166992188, "stability/token_entropy_mean": 4.91796875, "stability/repetition_rate_mean": 0.1269342005252838}
+{"current_steps": 90, "total_steps": 196, "loss": 0.5007, "lr": 8.9e-07, "epoch": 0.46065259117082535, "percentage": 45.92, "elapsed_time": "0:24:10", "remaining_time": "0:28:28", "rewards/chosen": 0.02295044293770423, "logps/chosen": -380.5598958333333, "logits/chosen": -58417874.05128205, "rewards/rejected": 0.007518144642434469, "logps/rejected": -187.15162919207316, "logits/rejected": -50577639.02439024, "rewards/margins": 0.015432298295269762, "regularization/mmd": 0.002058969810605049, "regularization/rkhs_norm": 2.058969736099243, "kl": 0.16505120694637299, "stability/response_length_mean": 32.875, "stability/response_length_std": 30.31000518798828, "stability/response_length_var": 918.6964111328125, "stability/token_entropy_mean": 3.001953125, "stability/repetition_rate_mean": 0.2901870310306549}
+{"current_steps": 95, "total_steps": 196, "loss": 0.5007, "lr": 9.399999999999999e-07, "epoch": 0.48624440179142675, "percentage": 48.47, "elapsed_time": "0:25:30", "remaining_time": "0:27:07", "rewards/chosen": 0.023500275008286102, "logps/chosen": -314.7583811313291, "logits/chosen": -50385446.88607595, "rewards/rejected": 0.007937841945224337, "logps/rejected": -207.0643325617284, "logits/rejected": -48970211.55555555, "rewards/margins": 0.015562433063061765, "regularization/mmd": 0.0023521382827311754, "regularization/rkhs_norm": 2.3521382808685303, "kl": 0.16731511056423187, "stability/response_length_mean": 27.0, "stability/response_length_std": 31.1264705657959, "stability/response_length_var": 968.8571166992188, "stability/token_entropy_mean": 3.689453125, "stability/repetition_rate_mean": 0.19764858484268188}
+{"current_steps": 100, "total_steps": 196, "loss": 0.5006, "lr": 9.9e-07, "epoch": 0.5118362124120281, "percentage": 51.02, "elapsed_time": "0:26:51", "remaining_time": "0:25:46", "rewards/chosen": 0.03295857197529561, "logps/chosen": -359.5405933277027, "logits/chosen": -54718024.64864865, "rewards/rejected": 0.01964006313057833, "logps/rejected": -216.3824945494186, "logits/rejected": -50911166.511627905, "rewards/margins": 0.013318508844717278, "regularization/mmd": 0.0024804905988276005, "regularization/rkhs_norm": 2.4804904460906982, "kl": 0.21206355094909668, "stability/response_length_mean": 38.0, "stability/response_length_std": 30.10457992553711, "stability/response_length_var": 906.2857055664062, "stability/token_entropy_mean": 4.142578125, "stability/repetition_rate_mean": 0.23201069235801697}
+{"current_steps": 105, "total_steps": 196, "loss": 0.5003, "lr": 9.999815554351224e-07, "epoch": 0.5374280230326296, "percentage": 53.57, "elapsed_time": "0:28:12", "remaining_time": "0:24:26", "rewards/chosen": 0.039128665464470185, "logps/chosen": -399.5112010542169, "logits/chosen": -59385825.15662651, "rewards/rejected": 0.017846293263621145, "logps/rejected": -209.8253601866883, "logits/rejected": -53806006.85714286, "rewards/margins": 0.02128237220084904, "regularization/mmd": 0.0032832843717187643, "regularization/rkhs_norm": 3.2832839488983154, "kl": 0.22991745173931122, "stability/response_length_mean": 29.0, "stability/response_length_std": 29.89505386352539, "stability/response_length_var": 893.7142944335938, "stability/token_entropy_mean": 3.376953125, "stability/repetition_rate_mean": 0.1858258843421936}
+{"current_steps": 110, "total_steps": 196, "loss": 0.5005, "lr": 9.999066267225444e-07, "epoch": 0.5630198336532309, "percentage": 56.12, "elapsed_time": "0:29:33", "remaining_time": "0:23:06", "rewards/chosen": 0.034520548510264204, "logps/chosen": -378.2180440512048, "logits/chosen": -57388278.746987954, "rewards/rejected": 0.017304101547637542, "logps/rejected": -205.9150897930195, "logits/rejected": -53613072.62337662, "rewards/margins": 0.017216446962626662, "regularization/mmd": 0.003176226047798991, "regularization/rkhs_norm": 3.1762256622314453, "kl": 0.2624301314353943, "stability/response_length_mean": 36.375, "stability/response_length_std": 30.302934646606445, "stability/response_length_var": 918.2678833007812, "stability/token_entropy_mean": 4.107421875, "stability/repetition_rate_mean": 0.30462831258773804}
+{"current_steps": 115, "total_steps": 196, "loss": 0.5002, "lr": 9.997740697079592e-07, "epoch": 0.5886116442738324, "percentage": 58.67, "elapsed_time": "0:30:55", "remaining_time": "0:21:46", "rewards/chosen": 0.05171521102325826, "logps/chosen": -364.07535601265823, "logits/chosen": -60608589.7721519, "rewards/rejected": 0.020484038341192553, "logps/rejected": -200.99171730324073, "logits/rejected": -53402333.2345679, "rewards/margins": 0.031231172682065707, "regularization/mmd": 0.0034977674949914217, "regularization/rkhs_norm": 3.497767210006714, "kl": 0.3131505250930786, "stability/response_length_mean": 41.375, "stability/response_length_std": 31.25442886352539, "stability/response_length_var": 976.8392944335938, "stability/token_entropy_mean": 4.931640625, "stability/repetition_rate_mean": 0.2838541865348816}
+{"current_steps": 120, "total_steps": 196, "loss": 0.4999, "lr": 9.995838996722914e-07, "epoch": 0.6142034548944337, "percentage": 61.22, "elapsed_time": "0:32:14", "remaining_time": "0:20:25", "rewards/chosen": 0.05807405982921326, "logps/chosen": -383.664164624183, "logits/chosen": -57739310.849673204, "rewards/rejected": 0.027071301808614216, "logps/rejected": -211.9456399700599, "logits/rejected": -54354569.964071855, "rewards/margins": 0.03100275802059904, "regularization/mmd": 0.004109646659344435, "regularization/rkhs_norm": 4.109646320343018, "kl": 0.35157909989356995, "stability/response_length_mean": 27.75, "stability/response_length_std": 30.358572006225586, "stability/response_length_var": 921.6428833007812, "stability/token_entropy_mean": 5.427734375, "stability/repetition_rate_mean": 0.16154514253139496}
+{"current_steps": 125, "total_steps": 196, "loss": 0.5007, "lr": 9.993361385379874e-07, "epoch": 0.6397952655150352, "percentage": 63.78, "elapsed_time": "0:33:35", "remaining_time": "0:19:04", "rewards/chosen": 0.05882363776638083, "logps/chosen": -416.791069135274, "logits/chosen": -60694717.36986301, "rewards/rejected": 0.027183488867748743, "logps/rejected": -216.09536637931035, "logits/rejected": -55665263.816091955, "rewards/margins": 0.03164014889863209, "regularization/mmd": 0.0047620246186852455, "regularization/rkhs_norm": 4.76202392578125, "kl": 0.40501686930656433, "stability/response_length_mean": 15.25, "stability/response_length_std": 24.93276596069336, "stability/response_length_var": 621.6428833007812, "stability/token_entropy_mean": 3.6484375, "stability/repetition_rate_mean": 0.15937499701976776}
+{"current_steps": 130, "total_steps": 196, "loss": 0.5002, "lr": 9.99030814866488e-07, "epoch": 0.6653870761356366, "percentage": 66.33, "elapsed_time": "0:34:55", "remaining_time": "0:17:44", "rewards/chosen": 0.06675009573659589, "logps/chosen": -391.99042338709677, "logits/chosen": -57384603.2516129, "rewards/rejected": 0.026037013891971474, "logps/rejected": -183.65854640151514, "logits/rejected": -53822941.86666667, "rewards/margins": 0.04071308184462442, "regularization/mmd": 0.005418591201305389, "regularization/rkhs_norm": 5.418591022491455, "kl": 0.44866982102394104, "stability/response_length_mean": 35.0, "stability/response_length_std": 27.250164031982422, "stability/response_length_var": 742.5714111328125, "stability/token_entropy_mean": 4.01171875, "stability/repetition_rate_mean": 0.20924006402492523}
+{"current_steps": 135, "total_steps": 196, "loss": 0.4997, "lr": 9.986679638549358e-07, "epoch": 0.690978886756238, "percentage": 68.88, "elapsed_time": "0:36:15", "remaining_time": "0:16:23", "rewards/chosen": 0.08449279457513541, "logps/chosen": -400.97124233128835, "logits/chosen": -59839180.17177914, "rewards/rejected": 0.03617833981848067, "logps/rejected": -247.50895700636943, "logits/rejected": -54987508.58598726, "rewards/margins": 0.04831445475665474, "regularization/mmd": 0.006465181242674589, "regularization/rkhs_norm": 6.465180397033691, "kl": 0.5167887806892395, "stability/response_length_mean": 29.25, "stability/response_length_std": 29.936838150024414, "stability/response_length_var": 896.2142944335938, "stability/token_entropy_mean": 4.078125, "stability/repetition_rate_mean": 0.2230902761220932}
+{"current_steps": 140, "total_steps": 196, "loss": 0.4993, "lr": 9.982476273321175e-07, "epoch": 0.7165706973768394, "percentage": 71.43, "elapsed_time": "0:37:36", "remaining_time": "0:15:02", "rewards/chosen": 0.08117941654089725, "logps/chosen": -366.377959280303, "logits/chosen": -57080863.03030303, "rewards/rejected": 0.01124683964637018, "logps/rejected": -189.92201360887097, "logits/rejected": -50078974.3483871, "rewards/margins": 0.06993257689452707, "regularization/mmd": 0.0070263417437672615, "regularization/rkhs_norm": 7.026341438293457, "kl": 0.48667556047439575, "stability/response_length_mean": 52.75, "stability/response_length_std": 22.72663688659668, "stability/response_length_var": 516.5, "stability/token_entropy_mean": 5.0244140625, "stability/repetition_rate_mean": 0.2858072817325592}
+{"current_steps": 145, "total_steps": 196, "loss": 0.4994, "lr": 9.977698537536417e-07, "epoch": 0.7421625079974408, "percentage": 73.98, "elapsed_time": "0:38:56", "remaining_time": "0:13:41", "rewards/chosen": 0.08052689196115517, "logps/chosen": -329.46656155873495, "logits/chosen": -51148756.81927711, "rewards/rejected": 0.033453972308666674, "logps/rejected": -199.1533710430195, "logits/rejected": -51663512.935064934, "rewards/margins": 0.0470729196524885, "regularization/mmd": 0.006148753222078085, "regularization/rkhs_norm": 6.148753643035889, "kl": 0.564969003200531, "stability/response_length_mean": 22.625, "stability/response_length_std": 26.359804153442383, "stability/response_length_var": 694.8392944335938, "stability/token_entropy_mean": 5.72265625, "stability/repetition_rate_mean": 0.18058820068836212}
+{"current_steps": 150, "total_steps": 196, "loss": 0.4998, "lr": 9.972346981963546e-07, "epoch": 0.7677543186180422, "percentage": 76.53, "elapsed_time": "0:40:17", "remaining_time": "0:12:21", "rewards/chosen": 0.11272701936609605, "logps/chosen": -396.80716911764705, "logits/chosen": -55423662.68235294, "rewards/rejected": 0.03462484359741211, "logps/rejected": -207.3796875, "logits/rejected": -50186574.50666667, "rewards/margins": 0.07810217576868395, "regularization/mmd": 0.007216709200292826, "regularization/rkhs_norm": 7.216709136962891, "kl": 0.6379441022872925, "stability/response_length_mean": 35.25, "stability/response_length_std": 31.01957893371582, "stability/response_length_var": 962.2142944335938, "stability/token_entropy_mean": 4.64453125, "stability/repetition_rate_mean": 0.2408854216337204}
+{"current_steps": 155, "total_steps": 196, "loss": 0.5004, "lr": 9.966422223519885e-07, "epoch": 0.7933461292386437, "percentage": 79.08, "elapsed_time": "0:41:39", "remaining_time": "0:11:01", "rewards/chosen": 0.08873968124389649, "logps/chosen": -360.1019071691176, "logits/chosen": -54007579.105882354, "rewards/rejected": 0.028631283442179363, "logps/rejected": -191.019296875, "logits/rejected": -49333650.77333333, "rewards/margins": 0.060108397801717124, "regularization/mmd": 0.008163990452885628, "regularization/rkhs_norm": 8.163989067077637, "kl": 0.6466606855392456, "stability/response_length_mean": 54.25, "stability/response_length_std": 20.457273483276367, "stability/response_length_var": 418.5, "stability/token_entropy_mean": 4.7119140625, "stability/repetition_rate_mean": 0.3057386875152588}
+{"current_steps": 160, "total_steps": 196, "loss": 0.4999, "lr": 9.959924945200523e-07, "epoch": 0.818937939859245, "percentage": 81.63, "elapsed_time": "0:43:00", "remaining_time": "0:09:40", "rewards/chosen": 0.13558682941255115, "logps/chosen": -415.5394810267857, "logits/chosen": -55253735.61904762, "rewards/rejected": 0.039586239739468225, "logps/rejected": -186.50149054276315, "logits/rejected": -48960865.684210524, "rewards/margins": 0.09600058967308292, "regularization/mmd": 0.00858414825052023, "regularization/rkhs_norm": 8.584148406982422, "kl": 0.7146582007408142, "stability/response_length_mean": 32.5, "stability/response_length_std": 30.528675079345703, "stability/response_length_var": 932.0, "stability/token_entropy_mean": 4.44921875, "stability/repetition_rate_mean": 0.30851560831069946}
+{"current_steps": 165, "total_steps": 196, "loss": 0.499, "lr": 9.952855895999568e-07, "epoch": 0.8445297504798465, "percentage": 84.18, "elapsed_time": "0:44:20", "remaining_time": "0:08:19", "rewards/chosen": 0.11838912963867188, "logps/chosen": -369.73284841954023, "logits/chosen": -54409180.68965517, "rewards/rejected": 0.04709646799792982, "logps/rejected": -212.12114726027397, "logits/rejected": -49792936.32876712, "rewards/margins": 0.07129266164074205, "regularization/mmd": 0.008565567433834076, "regularization/rkhs_norm": 8.565567016601562, "kl": 0.7538337707519531, "stability/response_length_mean": 22.5, "stability/response_length_std": 28.33977508544922, "stability/response_length_var": 803.1428833007812, "stability/token_entropy_mean": 4.85546875, "stability/repetition_rate_mean": 0.2411106526851654}
+{"current_steps": 170, "total_steps": 196, "loss": 0.4981, "lr": 9.9452158908238e-07, "epoch": 0.8701215611004478, "percentage": 86.73, "elapsed_time": "0:45:41", "remaining_time": "0:06:59", "rewards/chosen": 0.16798810563849273, "logps/chosen": -439.6619822485207, "logits/chosen": -59134242.840236686, "rewards/rejected": 0.06570145310155603, "logps/rejected": -190.60081229304635, "logits/rejected": -52560461.98675497, "rewards/margins": 0.1022866525369367, "regularization/mmd": 0.010343970730900764, "regularization/rkhs_norm": 10.34397029876709, "kl": 0.9214170575141907, "stability/response_length_mean": 44.125, "stability/response_length_std": 28.43256378173828, "stability/response_length_var": 808.4107055664062, "stability/token_entropy_mean": 3.58251953125, "stability/repetition_rate_mean": 0.3512404263019562}
+{"current_steps": 175, "total_steps": 196, "loss": 0.4983, "lr": 9.937005810398745e-07, "epoch": 0.8957133717210493, "percentage": 89.29, "elapsed_time": "0:47:02", "remaining_time": "0:05:38", "rewards/chosen": 0.15708599320377212, "logps/chosen": -354.86631682981925, "logits/chosen": -56639210.409638554, "rewards/rejected": 0.048284025935383584, "logps/rejected": -194.32470068993507, "logits/rejected": -48014429.09090909, "rewards/margins": 0.10880196726838853, "regularization/mmd": 0.012221619486808777, "regularization/rkhs_norm": 12.22161865234375, "kl": 0.9537512063980103, "stability/response_length_mean": 27.0, "stability/response_length_std": 25.773740768432617, "stability/response_length_var": 664.2857055664062, "stability/token_entropy_mean": 4.865234375, "stability/repetition_rate_mean": 0.18430830538272858}
+{"current_steps": 180, "total_steps": 196, "loss": 0.4982, "lr": 9.928226601167138e-07, "epoch": 0.9213051823416507, "percentage": 91.84, "elapsed_time": "0:48:22", "remaining_time": "0:04:17", "rewards/chosen": 0.1745892858212711, "logps/chosen": -374.39994727760734, "logits/chosen": -52637997.54601227, "rewards/rejected": 0.07463043966111105, "logps/rejected": -210.55190087579618, "logits/rejected": -47981639.74522293, "rewards/margins": 0.09995884616016004, "regularization/mmd": 0.012582081370055676, "regularization/rkhs_norm": 12.582079887390137, "kl": 1.0256928205490112, "stability/response_length_mean": 34.75, "stability/response_length_std": 31.54928207397461, "stability/response_length_var": 995.3571166992188, "stability/token_entropy_mean": 3.251953125, "stability/repetition_rate_mean": 0.25468748807907104}
+{"current_steps": 185, "total_steps": 196, "loss": 0.4988, "lr": 9.918879275179817e-07, "epoch": 0.946896992962252, "percentage": 94.39, "elapsed_time": "0:49:42", "remaining_time": "0:02:57", "rewards/chosen": 0.16850916544596353, "logps/chosen": -310.3162856867284, "logits/chosen": -54603801.28395062, "rewards/rejected": 0.047141669671746746, "logps/rejected": -196.56774129746836, "logits/rejected": -49515623.69620253, "rewards/margins": 0.1213674957742168, "regularization/mmd": 0.014902687631547451, "regularization/rkhs_norm": 14.90268611907959, "kl": 1.0763963460922241, "stability/response_length_mean": 32.875, "stability/response_length_std": 33.30996322631836, "stability/response_length_var": 1109.5535888671875, "stability/token_entropy_mean": 3.66015625, "stability/repetition_rate_mean": 0.197265625}
+{"current_steps": 190, "total_steps": 196, "loss": 0.4993, "lr": 9.90896490997906e-07, "epoch": 0.9724888035828535, "percentage": 96.94, "elapsed_time": "0:51:02", "remaining_time": "0:01:36", "rewards/chosen": 0.209123828453932, "logps/chosen": -390.3140671781437, "logits/chosen": -53093547.68862276, "rewards/rejected": 0.07299925610910055, "logps/rejected": -210.4685968137255, "logits/rejected": -49684118.5882353, "rewards/margins": 0.13612457234483144, "regularization/mmd": 0.016163000836968422, "regularization/rkhs_norm": 16.163000106811523, "kl": 1.1167560815811157, "stability/response_length_mean": 14.25, "stability/response_length_std": 21.743637084960938, "stability/response_length_var": 472.78570556640625, "stability/token_entropy_mean": 4.82421875, "stability/repetition_rate_mean": 0.1931029111146927}
+{"current_steps": 195, "total_steps": 196, "loss": 0.4996, "lr": 9.898484648474362e-07, "epoch": 0.9980806142034548, "percentage": 99.49, "elapsed_time": "0:52:23", "remaining_time": "0:00:16", "rewards/chosen": 0.1910219254032258, "logps/chosen": -390.87472278225806, "logits/chosen": -60031385.6, "rewards/rejected": 0.08385631098891749, "logps/rejected": -197.0807646780303, "logits/rejected": -53806036.557575755, "rewards/margins": 0.1071656144143083, "regularization/mmd": 0.013747951947152615, "regularization/rkhs_norm": 13.747952461242676, "kl": 1.1463875770568848, "stability/response_length_mean": 49.0, "stability/response_length_std": 27.82085609436035, "stability/response_length_var": 774.0, "stability/token_entropy_mean": 5.18359375, "stability/repetition_rate_mean": 0.3934151828289032}
+{"current_steps": 196, "total_steps": 196, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:52:36", "remaining_time": "0:00:00"}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/trainer_state.json

@@ -0,0 +1,974 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 196,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.025591810620601407,
+      "grad_norm": 0.5721998810768127,
+      "kl": 0.008188384585082531,
+      "learning_rate": 4e-08,
+      "logits/chosen": -59712215.578947365,
+      "logits/rejected": -50420742.0952381,
+      "logps/chosen": -438.99403782894734,
+      "logps/rejected": -198.40638950892858,
+      "loss": 0.5009,
+      "loss/base_kto": 0.4994224011898041,
+      "metrics/advantage_var": 0.04372745379805565,
+      "regularization/mmd": 0.0012165767839178443,
+      "regularization/rkhs_norm": 1.2165768146514893,
+      "rewards/chosen": 0.005160976397363763,
+      "rewards/margins": 0.004860207811931619,
+      "rewards/rejected": 0.00030076858543214344,
+      "step": 5
+    },
+    {
+      "epoch": 0.05118362124120281,
+      "grad_norm": 0.6033456921577454,
+      "kl": 0.017390631139278412,
+      "learning_rate": 9e-08,
+      "logits/chosen": -59291993.766233765,
+      "logits/rejected": -50017332.43373494,
+      "logps/chosen": -429.9233969155844,
+      "logps/rejected": -192.87807087725903,
+      "loss": 0.5018,
+      "loss/base_kto": 0.49968767166137695,
+      "metrics/advantage_var": 0.07829762995243073,
+      "regularization/mmd": 0.0020083182025700808,
+      "regularization/rkhs_norm": 2.0083181858062744,
+      "rewards/chosen": 0.001513990101876197,
+      "rewards/margins": 0.0025270523678583077,
+      "rewards/rejected": -0.0010130622659821108,
+      "stability/repetition_rate_mean": 0.3570815920829773,
+      "stability/response_length_mean": 46.125,
+      "stability/response_length_std": 27.231481552124023,
+      "stability/response_length_var": 741.5535888671875,
+      "stability/token_entropy_mean": 3.29638671875,
+      "step": 10
+    },
+    {
+      "epoch": 0.07677543186180422,
+      "grad_norm": 0.6098381876945496,
+      "kl": 0.00862761028110981,
+      "learning_rate": 1.4e-07,
+      "logits/chosen": -56235380.36363637,
+      "logits/rejected": -51927694.03870968,
+      "logps/chosen": -404.1193181818182,
+      "logps/rejected": -200.67879284274193,
+      "loss": 0.5016,
+      "loss/base_kto": 0.5000181198120117,
+      "metrics/advantage_var": 0.05759938433766365,
+      "regularization/mmd": 0.0016775511903688312,
+      "regularization/rkhs_norm": 1.6775509119033813,
+      "rewards/chosen": 0.0011517706242474644,
+      "rewards/margins": -0.0003764610311502584,
+      "rewards/rejected": 0.0015282316553977228,
+      "stability/repetition_rate_mean": 0.2651909589767456,
+      "stability/response_length_mean": 27.125,
+      "stability/response_length_std": 30.79163932800293,
+      "stability/response_length_var": 948.125,
+      "stability/token_entropy_mean": 3.82421875,
+      "step": 15
+    },
+    {
+      "epoch": 0.10236724248240563,
+      "grad_norm": 0.6189659237861633,
+      "kl": 0.011922934092581272,
+      "learning_rate": 1.8999999999999998e-07,
+      "logits/chosen": -51664051.2,
+      "logits/rejected": -48054336.0,
+      "logps/chosen": -343.67216796875,
+      "logps/rejected": -225.6396484375,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5000166296958923,
+      "metrics/advantage_var": 0.05863838270306587,
+      "regularization/mmd": 0.001691564917564392,
+      "regularization/rkhs_norm": 1.691564917564392,
+      "rewards/chosen": 0.002454766258597374,
+      "rewards/margins": 8.368939161300685e-05,
+      "rewards/rejected": 0.0023710768669843673,
+      "stability/repetition_rate_mean": 0.2155257910490036,
+      "stability/response_length_mean": 44.0,
+      "stability/response_length_std": 28.0611572265625,
+      "stability/response_length_var": 787.4285888671875,
+      "stability/token_entropy_mean": 4.42578125,
+      "step": 20
+    },
+    {
+      "epoch": 0.12795905310300704,
+      "grad_norm": 0.5778173804283142,
+      "kl": 0.018707742914557457,
+      "learning_rate": 2.4e-07,
+      "logits/chosen": -56476908.57931034,
+      "logits/rejected": -52024659.382857144,
+      "logps/chosen": -378.4651400862069,
+      "logps/rejected": -202.7953794642857,
+      "loss": 0.5019,
+      "loss/base_kto": 0.4996436536312103,
+      "metrics/advantage_var": 0.0549849271774292,
+      "regularization/mmd": 0.0017472734907642007,
+      "regularization/rkhs_norm": 1.7472734451293945,
+      "rewards/chosen": 0.0026301628556744806,
+      "rewards/margins": 0.0028841727326068974,
+      "rewards/rejected": -0.00025400987693241663,
+      "stability/repetition_rate_mean": 0.17606504261493683,
+      "stability/response_length_mean": 43.375,
+      "stability/response_length_std": 28.54538917541504,
+      "stability/response_length_var": 814.8392944335938,
+      "stability/token_entropy_mean": 4.763671875,
+      "step": 25
+    },
+    {
+      "epoch": 0.15355086372360843,
+      "grad_norm": 0.6033927202224731,
+      "kl": 0.017847830429673195,
+      "learning_rate": 2.9e-07,
+      "logits/chosen": -57730419.11409396,
+      "logits/rejected": -51521314.43274854,
+      "logps/chosen": -468.5690016778523,
+      "logps/rejected": -198.18969298245614,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5006142854690552,
+      "metrics/advantage_var": 0.08329301327466965,
+      "regularization/mmd": 0.0019578204955905676,
+      "regularization/rkhs_norm": 1.9578202962875366,
+      "rewards/chosen": 0.0012253856898954251,
+      "rewards/margins": -0.004593525076170741,
+      "rewards/rejected": 0.005818910766066166,
+      "stability/repetition_rate_mean": 0.1544828861951828,
+      "stability/response_length_mean": 43.0,
+      "stability/response_length_std": 29.602123260498047,
+      "stability/response_length_var": 876.2857055664062,
+      "stability/token_entropy_mean": 4.552734375,
+      "step": 30
+    },
+    {
+      "epoch": 0.17914267434420986,
+      "grad_norm": 0.6821840405464172,
+      "kl": 0.021474814042448997,
+      "learning_rate": 3.4000000000000003e-07,
+      "logits/chosen": -57633720.70886076,
+      "logits/rejected": -53879119.01234568,
+      "logps/chosen": -385.90676918512656,
+      "logps/rejected": -195.13110050154322,
+      "loss": 0.5017,
+      "loss/base_kto": 0.4995640218257904,
+      "metrics/advantage_var": 0.06586408615112305,
+      "regularization/mmd": 0.0018217876786366105,
+      "regularization/rkhs_norm": 1.821787714958191,
+      "rewards/chosen": 0.0029728393765944467,
+      "rewards/margins": 0.0034354667138971375,
+      "rewards/rejected": -0.0004626273373026907,
+      "stability/repetition_rate_mean": 0.34880331158638,
+      "stability/response_length_mean": 53.375,
+      "stability/response_length_std": 21.14533042907715,
+      "stability/response_length_var": 447.125,
+      "stability/token_entropy_mean": 5.326171875,
+      "step": 35
+    },
+    {
+      "epoch": 0.20473448496481125,
+      "grad_norm": 0.5855079293251038,
+      "kl": 0.018473921343684196,
+      "learning_rate": 3.8999999999999997e-07,
+      "logits/chosen": -56637171.2,
+      "logits/rejected": -50217504.0,
+      "logps/chosen": -421.93681640625,
+      "logps/rejected": -219.247998046875,
+      "loss": 0.5019,
+      "loss/base_kto": 0.5011991858482361,
+      "metrics/advantage_var": 0.08207302540540695,
+      "regularization/mmd": 0.0019648734014481306,
+      "regularization/rkhs_norm": 1.9648735523223877,
+      "rewards/chosen": -0.003416117653250694,
+      "rewards/margins": -0.009540746733546258,
+      "rewards/rejected": 0.006124629080295563,
+      "stability/repetition_rate_mean": 0.142578125,
+      "stability/response_length_mean": 33.125,
+      "stability/response_length_std": 33.01704025268555,
+      "stability/response_length_var": 1090.125,
+      "stability/token_entropy_mean": 4.9765625,
+      "step": 40
+    },
+    {
+      "epoch": 0.23032629558541268,
+      "grad_norm": 0.6406768560409546,
+      "kl": 0.032560475170612335,
+      "learning_rate": 4.3999999999999997e-07,
+      "logits/chosen": -52695545.12751678,
+      "logits/rejected": -50770458.94736842,
+      "logps/chosen": -352.29299496644296,
+      "logps/rejected": -243.82858187134502,
+      "loss": 0.5017,
+      "loss/base_kto": 0.5001169443130493,
+      "metrics/advantage_var": 0.062119271606206894,
+      "regularization/mmd": 0.0017398166237398982,
+      "regularization/rkhs_norm": 1.739816665649414,
+      "rewards/chosen": 0.0035465471696533614,
+      "rewards/margins": -0.0010111425584421558,
+      "rewards/rejected": 0.004557689728095517,
+      "stability/repetition_rate_mean": 0.1013549193739891,
+      "stability/response_length_mean": 29.25,
+      "stability/response_length_std": 29.51875877380371,
+      "stability/response_length_var": 871.3571166992188,
+      "stability/token_entropy_mean": 4.708984375,
+      "step": 45
+    },
+    {
+      "epoch": 0.2559181062060141,
+      "grad_norm": 0.5807592868804932,
+      "kl": 0.04730672389268875,
+      "learning_rate": 4.9e-07,
+      "logits/chosen": -54038620.77852349,
+      "logits/rejected": -48295651.55555555,
+      "logps/chosen": -375.1285654362416,
+      "logps/rejected": -180.20700840643275,
+      "loss": 0.5015,
+      "loss/base_kto": 0.4994547963142395,
+      "metrics/advantage_var": 0.06544163823127747,
+      "regularization/mmd": 0.0017960931872949004,
+      "regularization/rkhs_norm": 1.7960929870605469,
+      "rewards/chosen": 0.007055705025692114,
+      "rewards/margins": 0.0036947803606983817,
+      "rewards/rejected": 0.0033609246649937323,
+      "stability/repetition_rate_mean": 0.4108419120311737,
+      "stability/response_length_mean": 55.375,
+      "stability/response_length_std": 16.361431121826172,
+      "stability/response_length_var": 267.6964416503906,
+      "stability/token_entropy_mean": 3.814453125,
+      "step": 50
+    },
+    {
+      "epoch": 0.28150991682661547,
+      "grad_norm": 0.6596181988716125,
+      "kl": 0.05266108736395836,
+      "learning_rate": 5.4e-07,
+      "logits/chosen": -56956718.111801244,
+      "logits/rejected": -49555314.31446541,
+      "logps/chosen": -344.4710064052795,
+      "logps/rejected": -230.38991253930817,
+      "loss": 0.5015,
+      "loss/base_kto": 0.5002086162567139,
+      "metrics/advantage_var": 0.06512904167175293,
+      "regularization/mmd": 0.001749949180521071,
+      "regularization/rkhs_norm": 1.7499490976333618,
+      "rewards/chosen": 0.004102396668854707,
+      "rewards/margins": -0.0017093748379286402,
+      "rewards/rejected": 0.005811771506783347,
+      "stability/repetition_rate_mean": 0.10613363981246948,
+      "stability/response_length_mean": 22.375,
+      "stability/response_length_std": 30.236862182617188,
+      "stability/response_length_var": 914.2678833007812,
+      "stability/token_entropy_mean": 4.5703125,
+      "step": 55
+    },
+    {
+      "epoch": 0.30710172744721687,
+      "grad_norm": 0.5826641917228699,
+      "kl": 0.050893235951662064,
+      "learning_rate": 5.9e-07,
+      "logits/chosen": -52669142.10909091,
+      "logits/rejected": -47815765.88387097,
+      "logps/chosen": -359.6506628787879,
+      "logps/rejected": -178.1343245967742,
+      "loss": 0.5015,
+      "loss/base_kto": 0.5002776384353638,
+      "metrics/advantage_var": 0.07838872820138931,
+      "regularization/mmd": 0.0018729189177975059,
+      "regularization/rkhs_norm": 1.872918963432312,
+      "rewards/chosen": 0.007016492612434156,
+      "rewards/margins": -0.0022417499639188557,
+      "rewards/rejected": 0.009258242576353012,
+      "stability/repetition_rate_mean": 0.20835673809051514,
+      "stability/response_length_mean": 51.125,
+      "stability/response_length_std": 21.90523338317871,
+      "stability/response_length_var": 479.83929443359375,
+      "stability/token_entropy_mean": 5.353515625,
+      "step": 60
+    },
+    {
+      "epoch": 0.3326935380678183,
+      "grad_norm": 0.5758509039878845,
+      "kl": 0.06566640734672546,
+      "learning_rate": 6.4e-07,
+      "logits/chosen": -57555868.90322581,
+      "logits/rejected": -47274523.92727273,
+      "logps/chosen": -381.8256048387097,
+      "logps/rejected": -183.51074810606062,
+      "loss": 0.5013,
+      "loss/base_kto": 0.49938079714775085,
+      "metrics/advantage_var": 0.06294770538806915,
+      "regularization/mmd": 0.0016916670138016343,
+      "regularization/rkhs_norm": 1.6916671991348267,
+      "rewards/chosen": 0.012116812121483588,
+      "rewards/margins": 0.004956388380287447,
+      "rewards/rejected": 0.007160423741196141,
+      "stability/repetition_rate_mean": 0.18097305297851562,
+      "stability/response_length_mean": 17.875,
+      "stability/response_length_std": 23.485177993774414,
+      "stability/response_length_var": 551.5535888671875,
+      "stability/token_entropy_mean": 5.66796875,
+      "step": 65
+    },
+    {
+      "epoch": 0.3582853486884197,
+      "grad_norm": 0.5520736575126648,
+      "kl": 0.08278707414865494,
+      "learning_rate": 6.9e-07,
+      "logits/chosen": -52775096.980645165,
+      "logits/rejected": -49028474.56969697,
+      "logps/chosen": -378.99866431451613,
+      "logps/rejected": -220.35059185606062,
+      "loss": 0.5012,
+      "loss/base_kto": 0.4992857575416565,
+      "metrics/advantage_var": 0.06611553579568863,
+      "regularization/mmd": 0.001765405759215355,
+      "regularization/rkhs_norm": 1.7654056549072266,
+      "rewards/chosen": 0.01297938746790732,
+      "rewards/margins": 0.0058224533077325755,
+      "rewards/rejected": 0.007156934160174745,
+      "stability/repetition_rate_mean": 0.1875,
+      "stability/response_length_mean": 33.125,
+      "stability/response_length_std": 33.0300178527832,
+      "stability/response_length_var": 1090.982177734375,
+      "stability/token_entropy_mean": 5.765625,
+      "step": 70
+    },
+    {
+      "epoch": 0.3838771593090211,
+      "grad_norm": 0.6104229688644409,
+      "kl": 0.10445437580347061,
+      "learning_rate": 7.4e-07,
+      "logits/chosen": -49496012.10810811,
+      "logits/rejected": -47883255.069767445,
+      "logps/chosen": -314.14432010135135,
+      "logps/rejected": -228.81213662790697,
+      "loss": 0.5014,
+      "loss/base_kto": 0.4992406964302063,
+      "metrics/advantage_var": 0.07885061949491501,
+      "regularization/mmd": 0.0018627401441335678,
+      "regularization/rkhs_norm": 1.862740397453308,
+      "rewards/chosen": 0.012649115678426382,
+      "rewards/margins": 0.005379979937126021,
+      "rewards/rejected": 0.007269135741300361,
+      "stability/repetition_rate_mean": 0.06355167925357819,
+      "stability/response_length_mean": 22.75,
+      "stability/response_length_std": 31.07479476928711,
+      "stability/response_length_var": 965.6428833007812,
+      "stability/token_entropy_mean": 2.16015625,
+      "step": 75
+    },
+    {
+      "epoch": 0.4094689699296225,
+      "grad_norm": 0.6028388738632202,
+      "kl": 0.11964090168476105,
+      "learning_rate": 7.9e-07,
+      "logits/chosen": -58612227.39072847,
+      "logits/rejected": -53431647.43195266,
+      "logps/chosen": -403.8477597268212,
+      "logps/rejected": -200.98007581360946,
+      "loss": 0.5012,
+      "loss/base_kto": 0.4987204074859619,
+      "metrics/advantage_var": 0.07979276031255722,
+      "regularization/mmd": 0.0019349249778315425,
+      "regularization/rkhs_norm": 1.9349247217178345,
+      "rewards/chosen": 0.01823797131216289,
+      "rewards/margins": 0.010667078210724629,
+      "rewards/rejected": 0.007570893101438263,
+      "stability/repetition_rate_mean": 0.18697917461395264,
+      "stability/response_length_mean": 42.0,
+      "stability/response_length_std": 30.481843948364258,
+      "stability/response_length_var": 929.1428833007812,
+      "stability/token_entropy_mean": 5.25,
+      "step": 80
+    },
+    {
+      "epoch": 0.4350607805502239,
+      "grad_norm": 0.6117395758628845,
+      "kl": 0.12551236152648926,
+      "learning_rate": 8.399999999999999e-07,
+      "logits/chosen": -51474676.438709676,
+      "logits/rejected": -47137016.24242424,
+      "logps/chosen": -358.1219758064516,
+      "logps/rejected": -230.35598958333333,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49821534752845764,
+      "metrics/advantage_var": 0.09083787351846695,
+      "regularization/mmd": 0.0020756726153194904,
+      "regularization/rkhs_norm": 2.0756723880767822,
+      "rewards/chosen": 0.021053169619652534,
+      "rewards/margins": 0.014171496211375426,
+      "rewards/rejected": 0.006881673408277107,
+      "stability/repetition_rate_mean": 0.1269342005252838,
+      "stability/response_length_mean": 27.375,
+      "stability/response_length_std": 30.39707374572754,
+      "stability/response_length_var": 923.9821166992188,
+      "stability/token_entropy_mean": 4.91796875,
+      "step": 85
+    },
+    {
+      "epoch": 0.46065259117082535,
+      "grad_norm": 0.6396152973175049,
+      "kl": 0.16505120694637299,
+      "learning_rate": 8.9e-07,
+      "logits/chosen": -58417874.05128205,
+      "logits/rejected": -50577639.02439024,
+      "logps/chosen": -380.5598958333333,
+      "logps/rejected": -187.15162919207316,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49800893664360046,
+      "metrics/advantage_var": 0.09185568243265152,
+      "regularization/mmd": 0.002058969810605049,
+      "regularization/rkhs_norm": 2.058969736099243,
+      "rewards/chosen": 0.02295044293770423,
+      "rewards/margins": 0.015432298295269762,
+      "rewards/rejected": 0.007518144642434469,
+      "stability/repetition_rate_mean": 0.2901870310306549,
+      "stability/response_length_mean": 32.875,
+      "stability/response_length_std": 30.31000518798828,
+      "stability/response_length_var": 918.6964111328125,
+      "stability/token_entropy_mean": 3.001953125,
+      "step": 90
+    },
+    {
+      "epoch": 0.48624440179142675,
+      "grad_norm": 0.6152090430259705,
+      "kl": 0.16731511056423187,
+      "learning_rate": 9.399999999999999e-07,
+      "logits/chosen": -50385446.88607595,
+      "logits/rejected": -48970211.55555555,
+      "logps/chosen": -314.7583811313291,
+      "logps/rejected": -207.0643325617284,
+      "loss": 0.5007,
+      "loss/base_kto": 0.49809834361076355,
+      "metrics/advantage_var": 0.12706856429576874,
+      "regularization/mmd": 0.0023521382827311754,
+      "regularization/rkhs_norm": 2.3521382808685303,
+      "rewards/chosen": 0.023500275008286102,
+      "rewards/margins": 0.015562433063061765,
+      "rewards/rejected": 0.007937841945224337,
+      "stability/repetition_rate_mean": 0.19764858484268188,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 31.1264705657959,
+      "stability/response_length_var": 968.8571166992188,
+      "stability/token_entropy_mean": 3.689453125,
+      "step": 95
+    },
+    {
+      "epoch": 0.5118362124120281,
+      "grad_norm": 0.6100143790245056,
+      "kl": 0.21206355094909668,
+      "learning_rate": 9.9e-07,
+      "logits/chosen": -54718024.64864865,
+      "logits/rejected": -50911166.511627905,
+      "logps/chosen": -359.5405933277027,
+      "logps/rejected": -216.3824945494186,
+      "loss": 0.5006,
+      "loss/base_kto": 0.49836379289627075,
+      "metrics/advantage_var": 0.13836942613124847,
+      "regularization/mmd": 0.0024804905988276005,
+      "regularization/rkhs_norm": 2.4804904460906982,
+      "rewards/chosen": 0.03295857197529561,
+      "rewards/margins": 0.013318508844717278,
+      "rewards/rejected": 0.01964006313057833,
+      "stability/repetition_rate_mean": 0.23201069235801697,
+      "stability/response_length_mean": 38.0,
+      "stability/response_length_std": 30.10457992553711,
+      "stability/response_length_var": 906.2857055664062,
+      "stability/token_entropy_mean": 4.142578125,
+      "step": 100
+    },
+    {
+      "epoch": 0.5374280230326296,
+      "grad_norm": 0.6909270882606506,
+      "kl": 0.22991745173931122,
+      "learning_rate": 9.999815554351224e-07,
+      "logits/chosen": -59385825.15662651,
+      "logits/rejected": -53806006.85714286,
+      "logps/chosen": -399.5112010542169,
+      "logps/rejected": -209.8253601866883,
+      "loss": 0.5003,
+      "loss/base_kto": 0.4973313510417938,
+      "metrics/advantage_var": 0.24426975846290588,
+      "regularization/mmd": 0.0032832843717187643,
+      "regularization/rkhs_norm": 3.2832839488983154,
+      "rewards/chosen": 0.039128665464470185,
+      "rewards/margins": 0.02128237220084904,
+      "rewards/rejected": 0.017846293263621145,
+      "stability/repetition_rate_mean": 0.1858258843421936,
+      "stability/response_length_mean": 29.0,
+      "stability/response_length_std": 29.89505386352539,
+      "stability/response_length_var": 893.7142944335938,
+      "stability/token_entropy_mean": 3.376953125,
+      "step": 105
+    },
+    {
+      "epoch": 0.5630198336532309,
+      "grad_norm": 0.6516240835189819,
+      "kl": 0.2624301314353943,
+      "learning_rate": 9.999066267225444e-07,
+      "logits/chosen": -57388278.746987954,
+      "logits/rejected": -53613072.62337662,
+      "logps/chosen": -378.2180440512048,
+      "logps/rejected": -205.9150897930195,
+      "loss": 0.5005,
+      "loss/base_kto": 0.4977138042449951,
+      "metrics/advantage_var": 0.21244536340236664,
+      "regularization/mmd": 0.003176226047798991,
+      "regularization/rkhs_norm": 3.1762256622314453,
+      "rewards/chosen": 0.034520548510264204,
+      "rewards/margins": 0.017216446962626662,
+      "rewards/rejected": 0.017304101547637542,
+      "stability/repetition_rate_mean": 0.30462831258773804,
+      "stability/response_length_mean": 36.375,
+      "stability/response_length_std": 30.302934646606445,
+      "stability/response_length_var": 918.2678833007812,
+      "stability/token_entropy_mean": 4.107421875,
+      "step": 110
+    },
+    {
+      "epoch": 0.5886116442738324,
+      "grad_norm": 0.6488623023033142,
+      "kl": 0.3131505250930786,
+      "learning_rate": 9.997740697079592e-07,
+      "logits/chosen": -60608589.7721519,
+      "logits/rejected": -53402333.2345679,
+      "logps/chosen": -364.07535601265823,
+      "logps/rejected": -200.99171730324073,
+      "loss": 0.5002,
+      "loss/base_kto": 0.4960899353027344,
+      "metrics/advantage_var": 0.2569291293621063,
+      "regularization/mmd": 0.0034977674949914217,
+      "regularization/rkhs_norm": 3.497767210006714,
+      "rewards/chosen": 0.05171521102325826,
+      "rewards/margins": 0.031231172682065707,
+      "rewards/rejected": 0.020484038341192553,
+      "stability/repetition_rate_mean": 0.2838541865348816,
+      "stability/response_length_mean": 41.375,
+      "stability/response_length_std": 31.25442886352539,
+      "stability/response_length_var": 976.8392944335938,
+      "stability/token_entropy_mean": 4.931640625,
+      "step": 115
+    },
+    {
+      "epoch": 0.6142034548944337,
+      "grad_norm": 0.6185635924339294,
+      "kl": 0.35157909989356995,
+      "learning_rate": 9.995838996722914e-07,
+      "logits/chosen": -57739310.849673204,
+      "logits/rejected": -54354569.964071855,
+      "logps/chosen": -383.664164624183,
+      "logps/rejected": -211.9456399700599,
+      "loss": 0.4999,
+      "loss/base_kto": 0.4961109757423401,
+      "metrics/advantage_var": 0.4056810438632965,
+      "regularization/mmd": 0.004109646659344435,
+      "regularization/rkhs_norm": 4.109646320343018,
+      "rewards/chosen": 0.05807405982921326,
+      "rewards/margins": 0.03100275802059904,
+      "rewards/rejected": 0.027071301808614216,
+      "stability/repetition_rate_mean": 0.16154514253139496,
+      "stability/response_length_mean": 27.75,
+      "stability/response_length_std": 30.358572006225586,
+      "stability/response_length_var": 921.6428833007812,
+      "stability/token_entropy_mean": 5.427734375,
+      "step": 120
+    },
+    {
+      "epoch": 0.6397952655150352,
+      "grad_norm": 0.612278938293457,
+      "kl": 0.40501686930656433,
+      "learning_rate": 9.993361385379874e-07,
+      "logits/chosen": -60694717.36986301,
+      "logits/rejected": -55665263.816091955,
+      "logps/chosen": -416.791069135274,
+      "logps/rejected": -216.09536637931035,
+      "loss": 0.5007,
+      "loss/base_kto": 0.4961540400981903,
+      "metrics/advantage_var": 0.587132453918457,
+      "regularization/mmd": 0.0047620246186852455,
+      "regularization/rkhs_norm": 4.76202392578125,
+      "rewards/chosen": 0.05882363776638083,
+      "rewards/margins": 0.03164014889863209,
+      "rewards/rejected": 0.027183488867748743,
+      "stability/repetition_rate_mean": 0.15937499701976776,
+      "stability/response_length_mean": 15.25,
+      "stability/response_length_std": 24.93276596069336,
+      "stability/response_length_var": 621.6428833007812,
+      "stability/token_entropy_mean": 3.6484375,
+      "step": 125
+    },
+    {
+      "epoch": 0.6653870761356366,
+      "grad_norm": 0.5930764675140381,
+      "kl": 0.44866982102394104,
+      "learning_rate": 9.99030814866488e-07,
+      "logits/chosen": -57384603.2516129,
+      "logits/rejected": -53822941.86666667,
+      "logps/chosen": -391.99042338709677,
+      "logps/rejected": -183.65854640151514,
+      "loss": 0.5002,
+      "loss/base_kto": 0.49488160014152527,
+      "metrics/advantage_var": 0.7472460865974426,
+      "regularization/mmd": 0.005418591201305389,
+      "regularization/rkhs_norm": 5.418591022491455,
+      "rewards/chosen": 0.06675009573659589,
+      "rewards/margins": 0.04071308184462442,
+      "rewards/rejected": 0.026037013891971474,
+      "stability/repetition_rate_mean": 0.20924006402492523,
+      "stability/response_length_mean": 35.0,
+      "stability/response_length_std": 27.250164031982422,
+      "stability/response_length_var": 742.5714111328125,
+      "stability/token_entropy_mean": 4.01171875,
+      "step": 130
+    },
+    {
+      "epoch": 0.690978886756238,
+      "grad_norm": 0.5903881192207336,
+      "kl": 0.5167887806892395,
+      "learning_rate": 9.986679638549358e-07,
+      "logits/chosen": -59839180.17177914,
+      "logits/rejected": -54987508.58598726,
+      "logps/chosen": -400.97124233128835,
+      "logps/rejected": -247.50895700636943,
+      "loss": 0.4997,
+      "loss/base_kto": 0.4938087463378906,
+      "metrics/advantage_var": 0.9809302687644958,
+      "regularization/mmd": 0.006465181242674589,
+      "regularization/rkhs_norm": 6.465180397033691,
+      "rewards/chosen": 0.08449279457513541,
+      "rewards/margins": 0.04831445475665474,
+      "rewards/rejected": 0.03617833981848067,
+      "stability/repetition_rate_mean": 0.2230902761220932,
+      "stability/response_length_mean": 29.25,
+      "stability/response_length_std": 29.936838150024414,
+      "stability/response_length_var": 896.2142944335938,
+      "stability/token_entropy_mean": 4.078125,
+      "step": 135
+    },
+    {
+      "epoch": 0.7165706973768394,
+      "grad_norm": 0.7281087636947632,
+      "kl": 0.48667556047439575,
+      "learning_rate": 9.982476273321175e-07,
+      "logits/chosen": -57080863.03030303,
+      "logits/rejected": -50078974.3483871,
+      "logps/chosen": -366.377959280303,
+      "logps/rejected": -189.92201360887097,
+      "loss": 0.4993,
+      "loss/base_kto": 0.4917573034763336,
+      "metrics/advantage_var": 1.9869550466537476,
+      "regularization/mmd": 0.0070263417437672615,
+      "regularization/rkhs_norm": 7.026341438293457,
+      "rewards/chosen": 0.08117941654089725,
+      "rewards/margins": 0.06993257689452707,
+      "rewards/rejected": 0.01124683964637018,
+      "stability/repetition_rate_mean": 0.2858072817325592,
+      "stability/response_length_mean": 52.75,
+      "stability/response_length_std": 22.72663688659668,
+      "stability/response_length_var": 516.5,
+      "stability/token_entropy_mean": 5.0244140625,
+      "step": 140
+    },
+    {
+      "epoch": 0.7421625079974408,
+      "grad_norm": 0.6801453232765198,
+      "kl": 0.564969003200531,
+      "learning_rate": 9.977698537536417e-07,
+      "logits/chosen": -51148756.81927711,
+      "logits/rejected": -51663512.935064934,
+      "logps/chosen": -329.46656155873495,
+      "logps/rejected": -199.1533710430195,
+      "loss": 0.4994,
+      "loss/base_kto": 0.4940341114997864,
+      "metrics/advantage_var": 0.8375579714775085,
+      "regularization/mmd": 0.006148753222078085,
+      "regularization/rkhs_norm": 6.148753643035889,
+      "rewards/chosen": 0.08052689196115517,
+      "rewards/margins": 0.0470729196524885,
+      "rewards/rejected": 0.033453972308666674,
+      "stability/repetition_rate_mean": 0.18058820068836212,
+      "stability/response_length_mean": 22.625,
+      "stability/response_length_std": 26.359804153442383,
+      "stability/response_length_var": 694.8392944335938,
+      "stability/token_entropy_mean": 5.72265625,
+      "step": 145
+    },
+    {
+      "epoch": 0.7677543186180422,
+      "grad_norm": 0.6427031755447388,
+      "kl": 0.6379441022872925,
+      "learning_rate": 9.972346981963546e-07,
+      "logits/chosen": -55423662.68235294,
+      "logits/rejected": -50186574.50666667,
+      "logps/chosen": -396.80716911764705,
+      "logps/rejected": -207.3796875,
+      "loss": 0.4998,
+      "loss/base_kto": 0.4897971749305725,
+      "metrics/advantage_var": 1.2700830698013306,
+      "regularization/mmd": 0.007216709200292826,
+      "regularization/rkhs_norm": 7.216709136962891,
+      "rewards/chosen": 0.11272701936609605,
+      "rewards/margins": 0.07810217576868395,
+      "rewards/rejected": 0.03462484359741211,
+      "stability/repetition_rate_mean": 0.2408854216337204,
+      "stability/response_length_mean": 35.25,
+      "stability/response_length_std": 31.01957893371582,
+      "stability/response_length_var": 962.2142944335938,
+      "stability/token_entropy_mean": 4.64453125,
+      "step": 150
+    },
+    {
+      "epoch": 0.7933461292386437,
+      "grad_norm": 0.6279340982437134,
+      "kl": 0.6466606855392456,
+      "learning_rate": 9.966422223519885e-07,
+      "logits/chosen": -54007579.105882354,
+      "logits/rejected": -49333650.77333333,
+      "logps/chosen": -360.1019071691176,
+      "logps/rejected": -191.019296875,
+      "loss": 0.5004,
+      "loss/base_kto": 0.4928290843963623,
+      "metrics/advantage_var": 2.206279754638672,
+      "regularization/mmd": 0.008163990452885628,
+      "regularization/rkhs_norm": 8.163989067077637,
+      "rewards/chosen": 0.08873968124389649,
+      "rewards/margins": 0.060108397801717124,
+      "rewards/rejected": 0.028631283442179363,
+      "stability/repetition_rate_mean": 0.3057386875152588,
+      "stability/response_length_mean": 54.25,
+      "stability/response_length_std": 20.457273483276367,
+      "stability/response_length_var": 418.5,
+      "stability/token_entropy_mean": 4.7119140625,
+      "step": 155
+    },
+    {
+      "epoch": 0.818937939859245,
+      "grad_norm": 0.6161509156227112,
+      "kl": 0.7146582007408142,
+      "learning_rate": 9.959924945200523e-07,
+      "logits/chosen": -55253735.61904762,
+      "logits/rejected": -48960865.684210524,
+      "logps/chosen": -415.5394810267857,
+      "logps/rejected": -186.50149054276315,
+      "loss": 0.4999,
+      "loss/base_kto": 0.4877723157405853,
+      "metrics/advantage_var": 1.9780712127685547,
+      "regularization/mmd": 0.00858414825052023,
+      "regularization/rkhs_norm": 8.584148406982422,
+      "rewards/chosen": 0.13558682941255115,
+      "rewards/margins": 0.09600058967308292,
+      "rewards/rejected": 0.039586239739468225,
+      "stability/repetition_rate_mean": 0.30851560831069946,
+      "stability/response_length_mean": 32.5,
+      "stability/response_length_std": 30.528675079345703,
+      "stability/response_length_var": 932.0,
+      "stability/token_entropy_mean": 4.44921875,
+      "step": 160
+    },
+    {
+      "epoch": 0.8445297504798465,
+      "grad_norm": 0.7212228775024414,
+      "kl": 0.7538337707519531,
+      "learning_rate": 9.952855895999568e-07,
+      "logits/chosen": -54409180.68965517,
+      "logits/rejected": -49792936.32876712,
+      "logps/chosen": -369.73284841954023,
+      "logps/rejected": -212.12114726027397,
+      "loss": 0.499,
+      "loss/base_kto": 0.4910934567451477,
+      "metrics/advantage_var": 1.8075367212295532,
+      "regularization/mmd": 0.008565567433834076,
+      "regularization/rkhs_norm": 8.565567016601562,
+      "rewards/chosen": 0.11838912963867188,
+      "rewards/margins": 0.07129266164074205,
+      "rewards/rejected": 0.04709646799792982,
+      "stability/repetition_rate_mean": 0.2411106526851654,
+      "stability/response_length_mean": 22.5,
+      "stability/response_length_std": 28.33977508544922,
+      "stability/response_length_var": 803.1428833007812,
+      "stability/token_entropy_mean": 4.85546875,
+      "step": 165
+    },
+    {
+      "epoch": 0.8701215611004478,
+      "grad_norm": 0.6235238313674927,
+      "kl": 0.9214170575141907,
+      "learning_rate": 9.9452158908238e-07,
+      "logits/chosen": -59134242.840236686,
+      "logits/rejected": -52560461.98675497,
+      "logps/chosen": -439.6619822485207,
+      "logps/rejected": -190.60081229304635,
+      "loss": 0.4981,
+      "loss/base_kto": 0.4872615933418274,
+      "metrics/advantage_var": 2.389000654220581,
+      "regularization/mmd": 0.010343970730900764,
+      "regularization/rkhs_norm": 10.34397029876709,
+      "rewards/chosen": 0.16798810563849273,
+      "rewards/margins": 0.1022866525369367,
+      "rewards/rejected": 0.06570145310155603,
+      "stability/repetition_rate_mean": 0.3512404263019562,
+      "stability/response_length_mean": 44.125,
+      "stability/response_length_std": 28.43256378173828,
+      "stability/response_length_var": 808.4107055664062,
+      "stability/token_entropy_mean": 3.58251953125,
+      "step": 170
+    },
+    {
+      "epoch": 0.8957133717210493,
+      "grad_norm": 0.7214075922966003,
+      "kl": 0.9537512063980103,
+      "learning_rate": 9.937005810398745e-07,
+      "logits/chosen": -56639210.409638554,
+      "logits/rejected": -48014429.09090909,
+      "logps/chosen": -354.86631682981925,
+      "logps/rejected": -194.32470068993507,
+      "loss": 0.4983,
+      "loss/base_kto": 0.48649540543556213,
+      "metrics/advantage_var": 4.312074184417725,
+      "regularization/mmd": 0.012221619486808777,
+      "regularization/rkhs_norm": 12.22161865234375,
+      "rewards/chosen": 0.15708599320377212,
+      "rewards/margins": 0.10880196726838853,
+      "rewards/rejected": 0.048284025935383584,
+      "stability/repetition_rate_mean": 0.18430830538272858,
+      "stability/response_length_mean": 27.0,
+      "stability/response_length_std": 25.773740768432617,
+      "stability/response_length_var": 664.2857055664062,
+      "stability/token_entropy_mean": 4.865234375,
+      "step": 175
+    },
+    {
+      "epoch": 0.9213051823416507,
+      "grad_norm": 0.6424931287765503,
+      "kl": 1.0256928205490112,
+      "learning_rate": 9.928226601167138e-07,
+      "logits/chosen": -52637997.54601227,
+      "logits/rejected": -47981639.74522293,
+      "logps/chosen": -374.39994727760734,
+      "logps/rejected": -210.55190087579618,
+      "loss": 0.4982,
+      "loss/base_kto": 0.4874132573604584,
+      "metrics/advantage_var": 4.2638068199157715,
+      "regularization/mmd": 0.012582081370055676,
+      "regularization/rkhs_norm": 12.582079887390137,
+      "rewards/chosen": 0.1745892858212711,
+      "rewards/margins": 0.09995884616016004,
+      "rewards/rejected": 0.07463043966111105,
+      "stability/repetition_rate_mean": 0.25468748807907104,
+      "stability/response_length_mean": 34.75,
+      "stability/response_length_std": 31.54928207397461,
+      "stability/response_length_var": 995.3571166992188,
+      "stability/token_entropy_mean": 3.251953125,
+      "step": 180
+    },
+    {
+      "epoch": 0.946896992962252,
+      "grad_norm": 0.6472417116165161,
+      "kl": 1.0763963460922241,
+      "learning_rate": 9.918879275179817e-07,
+      "logits/chosen": -54603801.28395062,
+      "logits/rejected": -49515623.69620253,
+      "logps/chosen": -310.3162856867284,
+      "logps/rejected": -196.56774129746836,
+      "loss": 0.4988,
+      "loss/base_kto": 0.48661041259765625,
+      "metrics/advantage_var": 7.496189117431641,
+      "regularization/mmd": 0.014902687631547451,
+      "regularization/rkhs_norm": 14.90268611907959,
+      "rewards/chosen": 0.16850916544596353,
+      "rewards/margins": 0.1213674957742168,
+      "rewards/rejected": 0.047141669671746746,
+      "stability/repetition_rate_mean": 0.197265625,
+      "stability/response_length_mean": 32.875,
+      "stability/response_length_std": 33.30996322631836,
+      "stability/response_length_var": 1109.5535888671875,
+      "stability/token_entropy_mean": 3.66015625,
+      "step": 185
+    },
+    {
+      "epoch": 0.9724888035828535,
+      "grad_norm": 0.6970455646514893,
+      "kl": 1.1167560815811157,
+      "learning_rate": 9.90896490997906e-07,
+      "logits/chosen": -53093547.68862276,
+      "logits/rejected": -49684118.5882353,
+      "logps/chosen": -390.3140671781437,
+      "logps/rejected": -210.4685968137255,
+      "loss": 0.4993,
+      "loss/base_kto": 0.48344355821609497,
+      "metrics/advantage_var": 7.254169464111328,
+      "regularization/mmd": 0.016163000836968422,
+      "regularization/rkhs_norm": 16.163000106811523,
+      "rewards/chosen": 0.209123828453932,
+      "rewards/margins": 0.13612457234483144,
+      "rewards/rejected": 0.07299925610910055,
+      "stability/repetition_rate_mean": 0.1931029111146927,
+      "stability/response_length_mean": 14.25,
+      "stability/response_length_std": 21.743637084960938,
+      "stability/response_length_var": 472.78570556640625,
+      "stability/token_entropy_mean": 4.82421875,
+      "step": 190
+    },
+    {
+      "epoch": 0.9980806142034548,
+      "grad_norm": 0.6390827298164368,
+      "kl": 1.1463875770568848,
+      "learning_rate": 9.898484648474362e-07,
+      "logits/chosen": -60031385.6,
+      "logits/rejected": -53806036.557575755,
+      "logps/chosen": -390.87472278225806,
+      "logps/rejected": -197.0807646780303,
+      "loss": 0.4996,
+      "loss/base_kto": 0.48643895983695984,
+      "metrics/advantage_var": 4.362728118896484,
+      "regularization/mmd": 0.013747951947152615,
+      "regularization/rkhs_norm": 13.747952461242676,
+      "rewards/chosen": 0.1910219254032258,
+      "rewards/margins": 0.1071656144143083,
+      "rewards/rejected": 0.08385631098891749,
+      "stability/repetition_rate_mean": 0.3934151828289032,
+      "stability/response_length_mean": 49.0,
+      "stability/response_length_std": 27.82085609436035,
+      "stability/response_length_var": 774.0,
+      "stability/token_entropy_mean": 5.18359375,
+      "step": 195
+    },
+    {
+      "epoch": 1.0,
+      "step": 196,
+      "total_flos": 3.1861866257511875e+18,
+      "train_loss": 0.500500318955402,
+      "train_runtime": 3156.0327,
+      "train_samples_per_second": 31.685,
+      "train_steps_per_second": 0.062
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 196,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.1861866257511875e+18,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

mistral7b_kto_mmd_rho0.001_lambda_kr0.01/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da67b5a16dad8f0b2557e1d67989a1e387521af245b52b543853b881a35dbad4
+size 8081