Upload folder using huggingface_hub

.gitattributes

@@ -78,3 +78,47 @@ v4/DPO/DPO_5k/MDPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 v4/DPO/DPO_5k/lora/checkpoint-1000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 v4/DPO/DPO_5k/lora/checkpoint-1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 v4/DPO/DPO_5k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_10k/MDPO_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_10k/lora/checkpoint-1300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_10k/lora/checkpoint-2400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_1k/MDPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_1k/lora/checkpoint-240/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_1k/lora/checkpoint-250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_1k/lora/checkpoint-60/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_5k/MDPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_5k/lora/checkpoint-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_5k/lora/checkpoint-1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/DPO/DPO_5k/lora/checkpoint-300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_10k/MKTO_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_10k/lora/checkpoint-2300/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_10k/lora/checkpoint-2400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_10k/lora/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_1k/MKTO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_1k/lora/checkpoint-140/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_1k/lora/checkpoint-240/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_1k/lora/checkpoint-250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_20k/MKTO_20k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_20k/lora/checkpoint-4600/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_20k/lora/checkpoint-4800/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_20k/lora/checkpoint-5000/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_2k/MKTO_2k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_2k/lora/checkpoint-180/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_2k/lora/checkpoint-480/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_2k/lora/checkpoint-500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_5k/MKTO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_5k/lora/checkpoint-1150/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_5k/lora/checkpoint-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/KTO/KTO_5k/lora/checkpoint-1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_10k/MORPO_10k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_10k/lora/checkpoint-100/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_10k/lora/checkpoint-2400/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_10k/lora/checkpoint-2500/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_1k/MORPO_1k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_1k/lora/checkpoint-240/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_1k/lora/checkpoint-250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_1k/lora/checkpoint-30/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_5k/MORPO_5k/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_5k/lora/checkpoint-1200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_5k/lora/checkpoint-1250/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+v5/ORPO/ORPO_5k/lora/checkpoint-50/tokenizer.json filter=lfs diff=lfs merge=lfs -text

v5/DPO/DPO_10k/DPO_10k/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v5/DPO/DPO_10k/DPO_10k/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v5/DPO/DPO_10k/DPO_10k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ecfca13898e10dceef60271a2e863d7cc7700ce1555e77092c7927008052ac
+size 180385008

v5/DPO/DPO_10k/MDPO_10k/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v5/DPO/DPO_10k/MDPO_10k/config.json

@@ -0,0 +1,40 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "dtype": "float16",
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_theta": 500000.0,
+    "rope_type": "llama3"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_cache": true,
+  "vocab_size": 128256
+}

v5/DPO/DPO_10k/MDPO_10k/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "5.0.0"
+}

v5/DPO/DPO_10k/MDPO_10k/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c840c13f70893c0b8d55e3acd6553093375b17eaa47b583753de8b121643123
+size 2471645464

v5/DPO/DPO_10k/MDPO_10k/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v5/DPO/DPO_10k/MDPO_10k/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v5/DPO/DPO_10k/lora/README.md

@@ -0,0 +1,69 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: transformers
+model_name: lora
+tags:
+- generated_from_trainer
+- dpo
+- trl
+licence: license
+---
+
+# Model Card for lora
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-1B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/sea-rod/huggingface/runs/5573bwv9) 
+
+
+This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).
+
+### Framework versions
+
+- TRL: 0.27.2
+- Transformers: 5.0.0
+- Pytorch: 2.8.0+cu128
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite DPO as:
+
+```bibtex
+@inproceedings{rafailov2023direct,
+    title        = {{Direct Preference Optimization: Your Language Model is Secretly a Reward Model}},
+    author       = {Rafael Rafailov and Archit Sharma and Eric Mitchell and Christopher D. Manning and Stefano Ermon and Chelsea Finn},
+    year         = 2023,
+    booktitle    = {Advances in Neural Information Processing Systems 36: Annual Conference on Neural Information Processing Systems 2023, NeurIPS 2023, New Orleans, LA, USA, December 10 - 16, 2023},
+    url          = {http://papers.nips.cc/paper_files/paper/2023/hash/a85b405ed65c6477a4fe8302b5e06ce7-Abstract-Conference.html},
+    editor       = {Alice Oh and Tristan Naumann and Amir Globerson and Kate Saenko and Moritz Hardt and Sergey Levine},
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

v5/DPO/DPO_10k/lora/checkpoint-1300/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v5/DPO/DPO_10k/lora/checkpoint-1300/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v5/DPO/DPO_10k/lora/checkpoint-1300/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ecfca13898e10dceef60271a2e863d7cc7700ce1555e77092c7927008052ac
+size 180385008

v5/DPO/DPO_10k/lora/checkpoint-1300/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v5/DPO/DPO_10k/lora/checkpoint-1300/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71b5e638f59b9595d8dca87dcaa601984e35f6d6732e3de54770d1173653d04e
+size 360902475

v5/DPO/DPO_10k/lora/checkpoint-1300/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:95d744506ed8242dbe82c0f3357716f73248e5153ff68604326958faa28d9296
+size 14645

v5/DPO/DPO_10k/lora/checkpoint-1300/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6dc9658957f5cc4ceed4243b2de1cacb7762930be64bb5a8e59e057d65e5e2
+size 1383

v5/DPO/DPO_10k/lora/checkpoint-1300/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:619902af4750c8c03b81f910de02f255dd9ba762e9ef95909c545df05bfb4a75
+size 1465

v5/DPO/DPO_10k/lora/checkpoint-1300/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v5/DPO/DPO_10k/lora/checkpoint-1300/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v5/DPO/DPO_10k/lora/checkpoint-1300/trainer_state.json

@@ -0,0 +1,2192 @@
+{
+  "best_global_step": 1300,
+  "best_metric": 0.5460000038146973,
+  "best_model_checkpoint": "output/lora/checkpoint-1300",
+  "epoch": 1.04,
+  "eval_steps": 100,
+  "global_step": 1300,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008,
+      "grad_norm": 4.452983379364014,
+      "learning_rate": 3.6e-08,
+      "logits/chosen": 1.5510008335113525,
+      "logits/rejected": 1.5244438648223877,
+      "logps/chosen": -131.24708557128906,
+      "logps/rejected": -146.8297576904297,
+      "loss": 0.6932957172393799,
+      "rewards/accuracies": 0.1875,
+      "rewards/chosen": -0.00019072293071076274,
+      "rewards/margins": -0.00029331922996789217,
+      "rewards/rejected": 0.0001025962847052142,
+      "step": 10
+    },
+    {
+      "epoch": 0.016,
+      "grad_norm": 5.203515529632568,
+      "learning_rate": 7.599999999999999e-08,
+      "logits/chosen": 1.6611576080322266,
+      "logits/rejected": 1.6220839023590088,
+      "logps/chosen": -156.2080078125,
+      "logps/rejected": -142.72964477539062,
+      "loss": 0.6937986850738526,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0006145072402432561,
+      "rewards/margins": -0.0012890815269201994,
+      "rewards/rejected": 0.000674574519507587,
+      "step": 20
+    },
+    {
+      "epoch": 0.024,
+      "grad_norm": 4.087289810180664,
+      "learning_rate": 1.16e-07,
+      "logits/chosen": 1.9773778915405273,
+      "logits/rejected": 1.8304665088653564,
+      "logps/chosen": -163.54708862304688,
+      "logps/rejected": -157.88926696777344,
+      "loss": 0.6931437492370606,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0003584886435419321,
+      "rewards/margins": 2.1700874640373513e-05,
+      "rewards/rejected": 0.00033678775071166456,
+      "step": 30
+    },
+    {
+      "epoch": 0.032,
+      "grad_norm": 4.74172830581665,
+      "learning_rate": 1.56e-07,
+      "logits/chosen": 1.5896285772323608,
+      "logits/rejected": 1.7109922170639038,
+      "logps/chosen": -144.44276428222656,
+      "logps/rejected": -133.09629821777344,
+      "loss": 0.6932558059692383,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.00048673158744350076,
+      "rewards/margins": -0.00020531899644993246,
+      "rewards/rejected": 0.0006920504383742809,
+      "step": 40
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 4.32133150100708,
+      "learning_rate": 1.96e-07,
+      "logits/chosen": 1.5152148008346558,
+      "logits/rejected": 1.585367202758789,
+      "logps/chosen": -131.73226928710938,
+      "logps/rejected": -136.8301239013672,
+      "loss": 0.6930522918701172,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0008355070021934807,
+      "rewards/margins": 0.00019948731642216444,
+      "rewards/rejected": 0.0006360196857713163,
+      "step": 50
+    },
+    {
+      "epoch": 0.048,
+      "grad_norm": 3.915316343307495,
+      "learning_rate": 2.3599999999999997e-07,
+      "logits/chosen": 1.5138778686523438,
+      "logits/rejected": 1.4824903011322021,
+      "logps/chosen": -136.43399047851562,
+      "logps/rejected": -126.70623779296875,
+      "loss": 0.6929163455963134,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.000302538916002959,
+      "rewards/margins": 0.00047383070341311395,
+      "rewards/rejected": -0.00017129186016973108,
+      "step": 60
+    },
+    {
+      "epoch": 0.056,
+      "grad_norm": 4.329769134521484,
+      "learning_rate": 2.7600000000000004e-07,
+      "logits/chosen": 1.6920913457870483,
+      "logits/rejected": 1.8169019222259521,
+      "logps/chosen": -152.056640625,
+      "logps/rejected": -155.9404296875,
+      "loss": 0.6935575008392334,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0025545789394527674,
+      "rewards/margins": -0.0008073424687609076,
+      "rewards/rejected": 0.003361921291798353,
+      "step": 70
+    },
+    {
+      "epoch": 0.064,
+      "grad_norm": 3.964193344116211,
+      "learning_rate": 3.1599999999999997e-07,
+      "logits/chosen": 1.5645431280136108,
+      "logits/rejected": 1.5879082679748535,
+      "logps/chosen": -147.78839111328125,
+      "logps/rejected": -135.19906616210938,
+      "loss": 0.6925086498260498,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.006093275733292103,
+      "rewards/margins": 0.0012893510283902287,
+      "rewards/rejected": 0.004803924821317196,
+      "step": 80
+    },
+    {
+      "epoch": 0.072,
+      "grad_norm": 3.935694694519043,
+      "learning_rate": 3.5599999999999996e-07,
+      "logits/chosen": 1.5960246324539185,
+      "logits/rejected": 1.6901094913482666,
+      "logps/chosen": -157.85256958007812,
+      "logps/rejected": -150.51974487304688,
+      "loss": 0.6931850433349609,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.010219026356935501,
+      "rewards/margins": -5.4271204135147855e-05,
+      "rewards/rejected": 0.010273297317326069,
+      "step": 90
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 5.13019323348999,
+      "learning_rate": 3.96e-07,
+      "logits/chosen": 1.5446935892105103,
+      "logits/rejected": 1.6452451944351196,
+      "logps/chosen": -149.88038635253906,
+      "logps/rejected": -169.9078826904297,
+      "loss": 0.6935123443603516,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.011777262203395367,
+      "rewards/margins": -0.0007071519503369927,
+      "rewards/rejected": 0.0124844154343009,
+      "step": 100
+    },
+    {
+      "epoch": 0.08,
+      "eval_logits/chosen": 1.5421667098999023,
+      "eval_logits/rejected": 1.5734084844589233,
+      "eval_logps/chosen": -153.21649169921875,
+      "eval_logps/rejected": -147.7952117919922,
+      "eval_loss": 0.6929068565368652,
+      "eval_rewards/accuracies": 0.5180000066757202,
+      "eval_rewards/chosen": 0.011265883222222328,
+      "eval_rewards/margins": 0.0005034058121964335,
+      "eval_rewards/rejected": 0.010762478224933147,
+      "eval_runtime": 90.2131,
+      "eval_samples_per_second": 5.542,
+      "eval_steps_per_second": 2.771,
+      "step": 100
+    },
+    {
+      "epoch": 0.088,
+      "grad_norm": 5.695896625518799,
+      "learning_rate": 4.36e-07,
+      "logits/chosen": 1.781393051147461,
+      "logits/rejected": 1.7461833953857422,
+      "logps/chosen": -172.24188232421875,
+      "logps/rejected": -154.40878295898438,
+      "loss": 0.6922736167907715,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.014022141695022583,
+      "rewards/margins": 0.001777560799382627,
+      "rewards/rejected": 0.012244580313563347,
+      "step": 110
+    },
+    {
+      "epoch": 0.096,
+      "grad_norm": 4.398581027984619,
+      "learning_rate": 4.76e-07,
+      "logits/chosen": 1.611268401145935,
+      "logits/rejected": 1.6106624603271484,
+      "logps/chosen": -135.13426208496094,
+      "logps/rejected": -139.7284393310547,
+      "loss": 0.6927696228027344,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.014869053848087788,
+      "rewards/margins": 0.0007819391903467476,
+      "rewards/rejected": 0.014087115414440632,
+      "step": 120
+    },
+    {
+      "epoch": 0.104,
+      "grad_norm": 5.19202995300293,
+      "learning_rate": 5.16e-07,
+      "logits/chosen": 1.5615273714065552,
+      "logits/rejected": 1.7724416255950928,
+      "logps/chosen": -157.66746520996094,
+      "logps/rejected": -161.90391540527344,
+      "loss": 0.6928309917449951,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.013302234932780266,
+      "rewards/margins": 0.0006531739491038024,
+      "rewards/rejected": 0.012649061158299446,
+      "step": 130
+    },
+    {
+      "epoch": 0.112,
+      "grad_norm": 3.4575726985931396,
+      "learning_rate": 5.560000000000001e-07,
+      "logits/chosen": 1.492018699645996,
+      "logits/rejected": 1.5187314748764038,
+      "logps/chosen": -131.4152374267578,
+      "logps/rejected": -125.62705993652344,
+      "loss": 0.6929276943206787,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.014355423860251904,
+      "rewards/margins": 0.000462935131508857,
+      "rewards/rejected": 0.013892488554120064,
+      "step": 140
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 4.500187397003174,
+      "learning_rate": 5.96e-07,
+      "logits/chosen": 1.5862048864364624,
+      "logits/rejected": 1.6784181594848633,
+      "logps/chosen": -163.6667938232422,
+      "logps/rejected": -157.76402282714844,
+      "loss": 0.6910766124725342,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.018648523837327957,
+      "rewards/margins": 0.004183619283139706,
+      "rewards/rejected": 0.014464902691543102,
+      "step": 150
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 4.113079071044922,
+      "learning_rate": 6.36e-07,
+      "logits/chosen": 1.7717370986938477,
+      "logits/rejected": 1.8070589303970337,
+      "logps/chosen": -158.02734375,
+      "logps/rejected": -145.92495727539062,
+      "loss": 0.6927172183990479,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.02493301033973694,
+      "rewards/margins": 0.0009325124556198716,
+      "rewards/rejected": 0.024000495672225952,
+      "step": 160
+    },
+    {
+      "epoch": 0.136,
+      "grad_norm": 5.4983696937561035,
+      "learning_rate": 6.76e-07,
+      "logits/chosen": 1.5733931064605713,
+      "logits/rejected": 1.6008774042129517,
+      "logps/chosen": -147.1856689453125,
+      "logps/rejected": -159.93077087402344,
+      "loss": 0.6926907062530517,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.02877199277281761,
+      "rewards/margins": 0.0010106085101142526,
+      "rewards/rejected": 0.027761384844779968,
+      "step": 170
+    },
+    {
+      "epoch": 0.144,
+      "grad_norm": 4.50191068649292,
+      "learning_rate": 7.159999999999999e-07,
+      "logits/chosen": 1.5886398553848267,
+      "logits/rejected": 1.7399513721466064,
+      "logps/chosen": -157.6659393310547,
+      "logps/rejected": -160.65431213378906,
+      "loss": 0.6925735473632812,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.024526067078113556,
+      "rewards/margins": 0.0012190367560833693,
+      "rewards/rejected": 0.02330702915787697,
+      "step": 180
+    },
+    {
+      "epoch": 0.152,
+      "grad_norm": 4.708652019500732,
+      "learning_rate": 7.559999999999999e-07,
+      "logits/chosen": 1.6504443883895874,
+      "logits/rejected": 1.7761001586914062,
+      "logps/chosen": -141.34536743164062,
+      "logps/rejected": -143.41159057617188,
+      "loss": 0.6928653240203857,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.026504456996917725,
+      "rewards/margins": 0.0006459927535615861,
+      "rewards/rejected": 0.02585846558213234,
+      "step": 190
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.143187046051025,
+      "learning_rate": 7.96e-07,
+      "logits/chosen": 1.7108211517333984,
+      "logits/rejected": 1.6271438598632812,
+      "logps/chosen": -158.04931640625,
+      "logps/rejected": -132.23463439941406,
+      "loss": 0.6927096843719482,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.02748889848589897,
+      "rewards/margins": 0.0009879134595394135,
+      "rewards/rejected": 0.026500985026359558,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "eval_logits/chosen": 1.54364013671875,
+      "eval_logits/rejected": 1.5745173692703247,
+      "eval_logps/chosen": -153.04653930664062,
+      "eval_logps/rejected": -147.63844299316406,
+      "eval_loss": 0.6923084855079651,
+      "eval_rewards/accuracies": 0.5139999985694885,
+      "eval_rewards/chosen": 0.028259562328457832,
+      "eval_rewards/margins": 0.0018185621593147516,
+      "eval_rewards/rejected": 0.026441000401973724,
+      "eval_runtime": 90.4481,
+      "eval_samples_per_second": 5.528,
+      "eval_steps_per_second": 2.764,
+      "step": 200
+    },
+    {
+      "epoch": 0.168,
+      "grad_norm": 4.559652328491211,
+      "learning_rate": 8.359999999999999e-07,
+      "logits/chosen": 1.7295278310775757,
+      "logits/rejected": 1.6801897287368774,
+      "logps/chosen": -158.0893096923828,
+      "logps/rejected": -168.72427368164062,
+      "loss": 0.6922987461090088,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02837887406349182,
+      "rewards/margins": 0.0018027331680059433,
+      "rewards/rejected": 0.026576142758131027,
+      "step": 210
+    },
+    {
+      "epoch": 0.176,
+      "grad_norm": 3.907545328140259,
+      "learning_rate": 8.76e-07,
+      "logits/chosen": 1.6849712133407593,
+      "logits/rejected": 1.7441444396972656,
+      "logps/chosen": -158.67384338378906,
+      "logps/rejected": -143.02920532226562,
+      "loss": 0.6933117389678956,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03286944702267647,
+      "rewards/margins": -0.0002176974667236209,
+      "rewards/rejected": 0.03308714181184769,
+      "step": 220
+    },
+    {
+      "epoch": 0.184,
+      "grad_norm": 3.5083253383636475,
+      "learning_rate": 9.16e-07,
+      "logits/chosen": 1.5701725482940674,
+      "logits/rejected": 1.7182495594024658,
+      "logps/chosen": -160.56790161132812,
+      "logps/rejected": -138.05374145507812,
+      "loss": 0.6915814399719238,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.03840702772140503,
+      "rewards/margins": 0.0033035180531442165,
+      "rewards/rejected": 0.035103507339954376,
+      "step": 230
+    },
+    {
+      "epoch": 0.192,
+      "grad_norm": 4.424270153045654,
+      "learning_rate": 9.559999999999998e-07,
+      "logits/chosen": 1.671190857887268,
+      "logits/rejected": 1.6964585781097412,
+      "logps/chosen": -170.28260803222656,
+      "logps/rejected": -144.33534240722656,
+      "loss": 0.6900368690490722,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0511443130671978,
+      "rewards/margins": 0.006511001847684383,
+      "rewards/rejected": 0.04463331401348114,
+      "step": 240
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 4.5393967628479,
+      "learning_rate": 9.959999999999999e-07,
+      "logits/chosen": 1.6391950845718384,
+      "logits/rejected": 1.5815935134887695,
+      "logps/chosen": -160.45225524902344,
+      "logps/rejected": -147.56185913085938,
+      "loss": 0.6940414905548096,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.04959743469953537,
+      "rewards/margins": -0.001381749869324267,
+      "rewards/rejected": 0.05097918584942818,
+      "step": 250
+    },
+    {
+      "epoch": 0.208,
+      "grad_norm": 4.256033897399902,
+      "learning_rate": 9.959999999999999e-07,
+      "logits/chosen": 1.5204452276229858,
+      "logits/rejected": 1.6171140670776367,
+      "logps/chosen": -131.5397186279297,
+      "logps/rejected": -145.2186279296875,
+      "loss": 0.6930822372436524,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.045291412621736526,
+      "rewards/margins": 0.000405142258387059,
+      "rewards/rejected": 0.0448862686753273,
+      "step": 260
+    },
+    {
+      "epoch": 0.216,
+      "grad_norm": 4.027031421661377,
+      "learning_rate": 9.915555555555556e-07,
+      "logits/chosen": 1.6407123804092407,
+      "logits/rejected": 1.7262824773788452,
+      "logps/chosen": -145.78701782226562,
+      "logps/rejected": -146.34481811523438,
+      "loss": 0.6946187496185303,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.04104622080922127,
+      "rewards/margins": -0.0027089794166386127,
+      "rewards/rejected": 0.043755196034908295,
+      "step": 270
+    },
+    {
+      "epoch": 0.224,
+      "grad_norm": 5.568243026733398,
+      "learning_rate": 9.87111111111111e-07,
+      "logits/chosen": 1.6697533130645752,
+      "logits/rejected": 1.5154677629470825,
+      "logps/chosen": -171.0277099609375,
+      "logps/rejected": -154.05654907226562,
+      "loss": 0.6873753070831299,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04779375344514847,
+      "rewards/margins": 0.011815806850790977,
+      "rewards/rejected": 0.03597795218229294,
+      "step": 280
+    },
+    {
+      "epoch": 0.232,
+      "grad_norm": 4.041477680206299,
+      "learning_rate": 9.826666666666667e-07,
+      "logits/chosen": 1.6633354425430298,
+      "logits/rejected": 1.6905081272125244,
+      "logps/chosen": -142.88864135742188,
+      "logps/rejected": -152.04757690429688,
+      "loss": 0.6929869174957275,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.04824609309434891,
+      "rewards/margins": 0.0005596639821305871,
+      "rewards/rejected": 0.04768642783164978,
+      "step": 290
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 4.9481635093688965,
+      "learning_rate": 9.782222222222222e-07,
+      "logits/chosen": 1.617485761642456,
+      "logits/rejected": 1.6837307214736938,
+      "logps/chosen": -152.088134765625,
+      "logps/rejected": -164.15158081054688,
+      "loss": 0.6897455215454101,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.057097338140010834,
+      "rewards/margins": 0.007080497685819864,
+      "rewards/rejected": 0.050016842782497406,
+      "step": 300
+    },
+    {
+      "epoch": 0.24,
+      "eval_logits/chosen": 1.5520602464675903,
+      "eval_logits/rejected": 1.5826917886734009,
+      "eval_logps/chosen": -152.8227996826172,
+      "eval_logps/rejected": -147.43496704101562,
+      "eval_loss": 0.6914217472076416,
+      "eval_rewards/accuracies": 0.5260000228881836,
+      "eval_rewards/chosen": 0.05063560605049133,
+      "eval_rewards/margins": 0.0038486982230097055,
+      "eval_rewards/rejected": 0.04678690433502197,
+      "eval_runtime": 90.237,
+      "eval_samples_per_second": 5.541,
+      "eval_steps_per_second": 2.77,
+      "step": 300
+    },
+    {
+      "epoch": 0.248,
+      "grad_norm": 4.486109256744385,
+      "learning_rate": 9.737777777777777e-07,
+      "logits/chosen": 1.7188745737075806,
+      "logits/rejected": 1.7590553760528564,
+      "logps/chosen": -140.6877899169922,
+      "logps/rejected": -155.31893920898438,
+      "loss": 0.6952545166015625,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": 0.046565137803554535,
+      "rewards/margins": -0.003889651270583272,
+      "rewards/rejected": 0.050454795360565186,
+      "step": 310
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 4.374355316162109,
+      "learning_rate": 9.693333333333334e-07,
+      "logits/chosen": 1.7103513479232788,
+      "logits/rejected": 1.7379261255264282,
+      "logps/chosen": -137.8660888671875,
+      "logps/rejected": -140.40956115722656,
+      "loss": 0.692354393005371,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04029277712106705,
+      "rewards/margins": 0.0018124934285879135,
+      "rewards/rejected": 0.038480278104543686,
+      "step": 320
+    },
+    {
+      "epoch": 0.264,
+      "grad_norm": 4.618821144104004,
+      "learning_rate": 9.648888888888889e-07,
+      "logits/chosen": 1.5603777170181274,
+      "logits/rejected": 1.5868213176727295,
+      "logps/chosen": -157.3379669189453,
+      "logps/rejected": -182.77377319335938,
+      "loss": 0.6921377182006836,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.03832743316888809,
+      "rewards/margins": 0.0022720033302903175,
+      "rewards/rejected": 0.03605542704463005,
+      "step": 330
+    },
+    {
+      "epoch": 0.272,
+      "grad_norm": 4.1974406242370605,
+      "learning_rate": 9.604444444444443e-07,
+      "logits/chosen": 1.8520517349243164,
+      "logits/rejected": 1.716774582862854,
+      "logps/chosen": -158.16363525390625,
+      "logps/rejected": -149.66162109375,
+      "loss": 0.692081069946289,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0396885983645916,
+      "rewards/margins": 0.0023556998930871487,
+      "rewards/rejected": 0.03733289986848831,
+      "step": 340
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 4.700806140899658,
+      "learning_rate": 9.559999999999998e-07,
+      "logits/chosen": 1.6886537075042725,
+      "logits/rejected": 1.8079423904418945,
+      "logps/chosen": -165.5438232421875,
+      "logps/rejected": -194.98428344726562,
+      "loss": 0.6890993118286133,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.05031610652804375,
+      "rewards/margins": 0.008422226645052433,
+      "rewards/rejected": 0.041893888264894485,
+      "step": 350
+    },
+    {
+      "epoch": 0.288,
+      "grad_norm": 3.7786731719970703,
+      "learning_rate": 9.515555555555555e-07,
+      "logits/chosen": 1.5483795404434204,
+      "logits/rejected": 1.4731425046920776,
+      "logps/chosen": -161.77774047851562,
+      "logps/rejected": -168.05458068847656,
+      "loss": 0.6911417007446289,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.050054989755153656,
+      "rewards/margins": 0.0043535237200558186,
+      "rewards/rejected": 0.045701466500759125,
+      "step": 360
+    },
+    {
+      "epoch": 0.296,
+      "grad_norm": 4.362200736999512,
+      "learning_rate": 9.471111111111111e-07,
+      "logits/chosen": 1.735099196434021,
+      "logits/rejected": 1.7567729949951172,
+      "logps/chosen": -161.30374145507812,
+      "logps/rejected": -153.4731903076172,
+      "loss": 0.6882652282714844,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06078929826617241,
+      "rewards/margins": 0.010189466178417206,
+      "rewards/rejected": 0.0505998320877552,
+      "step": 370
+    },
+    {
+      "epoch": 0.304,
+      "grad_norm": 4.35581111907959,
+      "learning_rate": 9.426666666666666e-07,
+      "logits/chosen": 1.604020118713379,
+      "logits/rejected": 1.524717926979065,
+      "logps/chosen": -141.42324829101562,
+      "logps/rejected": -151.82521057128906,
+      "loss": 0.6888412952423095,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.06947806477546692,
+      "rewards/margins": 0.009350637905299664,
+      "rewards/rejected": 0.06012742593884468,
+      "step": 380
+    },
+    {
+      "epoch": 0.312,
+      "grad_norm": 4.360926628112793,
+      "learning_rate": 9.382222222222222e-07,
+      "logits/chosen": 1.7075706720352173,
+      "logits/rejected": 1.6819493770599365,
+      "logps/chosen": -150.63375854492188,
+      "logps/rejected": -137.05673217773438,
+      "loss": 0.6836989879608154,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.08739721775054932,
+      "rewards/margins": 0.019574418663978577,
+      "rewards/rejected": 0.06782279908657074,
+      "step": 390
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 4.909813404083252,
+      "learning_rate": 9.337777777777778e-07,
+      "logits/chosen": 1.6920162439346313,
+      "logits/rejected": 1.675100326538086,
+      "logps/chosen": -154.39663696289062,
+      "logps/rejected": -147.51455688476562,
+      "loss": 0.6892420768737793,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.08908190578222275,
+      "rewards/margins": 0.008944114670157433,
+      "rewards/rejected": 0.08013778924942017,
+      "step": 400
+    },
+    {
+      "epoch": 0.32,
+      "eval_logits/chosen": 1.5610027313232422,
+      "eval_logits/rejected": 1.5909091234207153,
+      "eval_logps/chosen": -152.39871215820312,
+      "eval_logps/rejected": -147.04049682617188,
+      "eval_loss": 0.6903401017189026,
+      "eval_rewards/accuracies": 0.5400000214576721,
+      "eval_rewards/chosen": 0.09304190427064896,
+      "eval_rewards/margins": 0.006809028796851635,
+      "eval_rewards/rejected": 0.08623287081718445,
+      "eval_runtime": 90.296,
+      "eval_samples_per_second": 5.537,
+      "eval_steps_per_second": 2.769,
+      "step": 400
+    },
+    {
+      "epoch": 0.328,
+      "grad_norm": 4.541158199310303,
+      "learning_rate": 9.293333333333333e-07,
+      "logits/chosen": 1.6515496969223022,
+      "logits/rejected": 1.548688530921936,
+      "logps/chosen": -147.21546936035156,
+      "logps/rejected": -187.50816345214844,
+      "loss": 0.6924624919891358,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.10388608276844025,
+      "rewards/margins": 0.002456969814375043,
+      "rewards/rejected": 0.10142910480499268,
+      "step": 410
+    },
+    {
+      "epoch": 0.336,
+      "grad_norm": 5.246954917907715,
+      "learning_rate": 9.248888888888888e-07,
+      "logits/chosen": 1.6460405588150024,
+      "logits/rejected": 1.6713184118270874,
+      "logps/chosen": -151.11341857910156,
+      "logps/rejected": -166.2979736328125,
+      "loss": 0.6986268043518067,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.09254096448421478,
+      "rewards/margins": -0.009933066554367542,
+      "rewards/rejected": 0.10247401893138885,
+      "step": 420
+    },
+    {
+      "epoch": 0.344,
+      "grad_norm": 4.068811416625977,
+      "learning_rate": 9.204444444444443e-07,
+      "logits/chosen": 1.6973702907562256,
+      "logits/rejected": 1.7605253458023071,
+      "logps/chosen": -162.7523651123047,
+      "logps/rejected": -150.79718017578125,
+      "loss": 0.6896752834320068,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.08682320266962051,
+      "rewards/margins": 0.007874277420341969,
+      "rewards/rejected": 0.07894892990589142,
+      "step": 430
+    },
+    {
+      "epoch": 0.352,
+      "grad_norm": 4.387909412384033,
+      "learning_rate": 9.16e-07,
+      "logits/chosen": 1.5456931591033936,
+      "logits/rejected": 1.4381892681121826,
+      "logps/chosen": -155.777099609375,
+      "logps/rejected": -144.95742797851562,
+      "loss": 0.6881390571594238,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.08465877920389175,
+      "rewards/margins": 0.01117948442697525,
+      "rewards/rejected": 0.0734792947769165,
+      "step": 440
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 4.3955864906311035,
+      "learning_rate": 9.115555555555555e-07,
+      "logits/chosen": 1.7298389673233032,
+      "logits/rejected": 1.681171178817749,
+      "logps/chosen": -156.2227783203125,
+      "logps/rejected": -158.81114196777344,
+      "loss": 0.685992956161499,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.08095243573188782,
+      "rewards/margins": 0.015426402911543846,
+      "rewards/rejected": 0.06552603840827942,
+      "step": 450
+    },
+    {
+      "epoch": 0.368,
+      "grad_norm": 4.6138176918029785,
+      "learning_rate": 9.071111111111111e-07,
+      "logits/chosen": 1.6315510272979736,
+      "logits/rejected": 1.6908462047576904,
+      "logps/chosen": -150.84512329101562,
+      "logps/rejected": -163.89492797851562,
+      "loss": 0.6891673088073731,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07254813611507416,
+      "rewards/margins": 0.008895034901797771,
+      "rewards/rejected": 0.06365309655666351,
+      "step": 460
+    },
+    {
+      "epoch": 0.376,
+      "grad_norm": 4.3172101974487305,
+      "learning_rate": 9.026666666666665e-07,
+      "logits/chosen": 1.4395225048065186,
+      "logits/rejected": 1.4489599466323853,
+      "logps/chosen": -130.1565399169922,
+      "logps/rejected": -122.24504089355469,
+      "loss": 0.6887143135070801,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.08355607837438583,
+      "rewards/margins": 0.009718736633658409,
+      "rewards/rejected": 0.07383735477924347,
+      "step": 470
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 4.2122087478637695,
+      "learning_rate": 8.982222222222222e-07,
+      "logits/chosen": 1.5334614515304565,
+      "logits/rejected": 1.5769469738006592,
+      "logps/chosen": -147.21896362304688,
+      "logps/rejected": -162.89804077148438,
+      "loss": 0.6849615573883057,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.10068760812282562,
+      "rewards/margins": 0.017685385420918465,
+      "rewards/rejected": 0.0830022394657135,
+      "step": 480
+    },
+    {
+      "epoch": 0.392,
+      "grad_norm": 4.740354061126709,
+      "learning_rate": 8.937777777777777e-07,
+      "logits/chosen": 1.6524379253387451,
+      "logits/rejected": 1.7100518941879272,
+      "logps/chosen": -142.10653686523438,
+      "logps/rejected": -158.3316192626953,
+      "loss": 0.696216630935669,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.08680267632007599,
+      "rewards/margins": -0.004664557985961437,
+      "rewards/rejected": 0.0914672389626503,
+      "step": 490
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 3.6374881267547607,
+      "learning_rate": 8.893333333333333e-07,
+      "logits/chosen": 1.518328309059143,
+      "logits/rejected": 1.6029644012451172,
+      "logps/chosen": -143.19154357910156,
+      "logps/rejected": -134.3892059326172,
+      "loss": 0.6908615589141845,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.06735874712467194,
+      "rewards/margins": 0.005541653838008642,
+      "rewards/rejected": 0.06181709095835686,
+      "step": 500
+    },
+    {
+      "epoch": 0.4,
+      "eval_logits/chosen": 1.5418404340744019,
+      "eval_logits/rejected": 1.571341633796692,
+      "eval_logps/chosen": -152.5870819091797,
+      "eval_logps/rejected": -147.23146057128906,
+      "eval_loss": 0.6903930902481079,
+      "eval_rewards/accuracies": 0.5220000147819519,
+      "eval_rewards/chosen": 0.07420650124549866,
+      "eval_rewards/margins": 0.007066408637911081,
+      "eval_rewards/rejected": 0.06714009493589401,
+      "eval_runtime": 90.217,
+      "eval_samples_per_second": 5.542,
+      "eval_steps_per_second": 2.771,
+      "step": 500
+    },
+    {
+      "epoch": 0.408,
+      "grad_norm": 4.53076171875,
+      "learning_rate": 8.848888888888888e-07,
+      "logits/chosen": 1.6947540044784546,
+      "logits/rejected": 1.6306483745574951,
+      "logps/chosen": -130.33372497558594,
+      "logps/rejected": -139.05648803710938,
+      "loss": 0.6863756656646729,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.08373989164829254,
+      "rewards/margins": 0.014761297032237053,
+      "rewards/rejected": 0.06897859275341034,
+      "step": 510
+    },
+    {
+      "epoch": 0.416,
+      "grad_norm": 5.064472675323486,
+      "learning_rate": 8.804444444444445e-07,
+      "logits/chosen": 1.795907974243164,
+      "logits/rejected": 1.6805435419082642,
+      "logps/chosen": -165.10183715820312,
+      "logps/rejected": -170.87112426757812,
+      "loss": 0.6915029525756836,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.07381857931613922,
+      "rewards/margins": 0.00408085435628891,
+      "rewards/rejected": 0.06973771750926971,
+      "step": 520
+    },
+    {
+      "epoch": 0.424,
+      "grad_norm": 4.472287178039551,
+      "learning_rate": 8.76e-07,
+      "logits/chosen": 1.7226626873016357,
+      "logits/rejected": 1.6465301513671875,
+      "logps/chosen": -165.50076293945312,
+      "logps/rejected": -167.12991333007812,
+      "loss": 0.6784487724304199,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.09387621283531189,
+      "rewards/margins": 0.031213903799653053,
+      "rewards/rejected": 0.06266231089830399,
+      "step": 530
+    },
+    {
+      "epoch": 0.432,
+      "grad_norm": 4.193634033203125,
+      "learning_rate": 8.715555555555554e-07,
+      "logits/chosen": 1.7823143005371094,
+      "logits/rejected": 1.7374283075332642,
+      "logps/chosen": -180.05233764648438,
+      "logps/rejected": -157.24835205078125,
+      "loss": 0.6891638278961182,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10219593346118927,
+      "rewards/margins": 0.009201721288263798,
+      "rewards/rejected": 0.09299422055482864,
+      "step": 540
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 5.877465724945068,
+      "learning_rate": 8.671111111111111e-07,
+      "logits/chosen": 1.6558294296264648,
+      "logits/rejected": 1.7549035549163818,
+      "logps/chosen": -149.97171020507812,
+      "logps/rejected": -166.52127075195312,
+      "loss": 0.6909477233886718,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.09389887005090714,
+      "rewards/margins": 0.00655100354924798,
+      "rewards/rejected": 0.0873478576540947,
+      "step": 550
+    },
+    {
+      "epoch": 0.448,
+      "grad_norm": 3.9154303073883057,
+      "learning_rate": 8.626666666666666e-07,
+      "logits/chosen": 1.7343839406967163,
+      "logits/rejected": 1.6256252527236938,
+      "logps/chosen": -153.2657470703125,
+      "logps/rejected": -137.84548950195312,
+      "loss": 0.6832056045532227,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.08965932577848434,
+      "rewards/margins": 0.02167549543082714,
+      "rewards/rejected": 0.06798382848501205,
+      "step": 560
+    },
+    {
+      "epoch": 0.456,
+      "grad_norm": 3.569357395172119,
+      "learning_rate": 8.582222222222222e-07,
+      "logits/chosen": 1.6020238399505615,
+      "logits/rejected": 1.5468555688858032,
+      "logps/chosen": -156.9928741455078,
+      "logps/rejected": -150.9776153564453,
+      "loss": 0.6857921123504639,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07846538722515106,
+      "rewards/margins": 0.016974106431007385,
+      "rewards/rejected": 0.06149129942059517,
+      "step": 570
+    },
+    {
+      "epoch": 0.464,
+      "grad_norm": 5.710695266723633,
+      "learning_rate": 8.537777777777777e-07,
+      "logits/chosen": 1.4293699264526367,
+      "logits/rejected": 1.583032250404358,
+      "logps/chosen": -134.98165893554688,
+      "logps/rejected": -153.61439514160156,
+      "loss": 0.6899324417114258,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.05123991519212723,
+      "rewards/margins": 0.008615568280220032,
+      "rewards/rejected": 0.042624346911907196,
+      "step": 580
+    },
+    {
+      "epoch": 0.472,
+      "grad_norm": 3.757844924926758,
+      "learning_rate": 8.493333333333334e-07,
+      "logits/chosen": 1.5719819068908691,
+      "logits/rejected": 1.5706799030303955,
+      "logps/chosen": -143.9678955078125,
+      "logps/rejected": -130.64585876464844,
+      "loss": 0.6851204395294189,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.07710663974285126,
+      "rewards/margins": 0.01796458289027214,
+      "rewards/rejected": 0.059142060577869415,
+      "step": 590
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 5.662181854248047,
+      "learning_rate": 8.448888888888888e-07,
+      "logits/chosen": 1.6224644184112549,
+      "logits/rejected": 1.6623615026474,
+      "logps/chosen": -130.7429962158203,
+      "logps/rejected": -157.59295654296875,
+      "loss": 0.6958520889282227,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0648646205663681,
+      "rewards/margins": -0.003241670085117221,
+      "rewards/rejected": 0.0681062787771225,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_logits/chosen": 1.5268914699554443,
+      "eval_logits/rejected": 1.5556302070617676,
+      "eval_logps/chosen": -152.5731201171875,
+      "eval_logps/rejected": -147.22303771972656,
+      "eval_loss": 0.6907246708869934,
+      "eval_rewards/accuracies": 0.5139999985694885,
+      "eval_rewards/chosen": 0.07560181617736816,
+      "eval_rewards/margins": 0.007620053365826607,
+      "eval_rewards/rejected": 0.0679817646741867,
+      "eval_runtime": 90.3327,
+      "eval_samples_per_second": 5.535,
+      "eval_steps_per_second": 2.768,
+      "step": 600
+    },
+    {
+      "epoch": 0.488,
+      "grad_norm": 3.7953426837921143,
+      "learning_rate": 8.404444444444444e-07,
+      "logits/chosen": 1.6380192041397095,
+      "logits/rejected": 1.6921494007110596,
+      "logps/chosen": -130.59445190429688,
+      "logps/rejected": -148.48709106445312,
+      "loss": 0.6903901100158691,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.08007006347179413,
+      "rewards/margins": 0.00720745325088501,
+      "rewards/rejected": 0.07286261022090912,
+      "step": 610
+    },
+    {
+      "epoch": 0.496,
+      "grad_norm": 7.121775150299072,
+      "learning_rate": 8.359999999999999e-07,
+      "logits/chosen": 1.6000845432281494,
+      "logits/rejected": 1.731951355934143,
+      "logps/chosen": -154.8905792236328,
+      "logps/rejected": -166.4490966796875,
+      "loss": 0.6969138145446777,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.05843223258852959,
+      "rewards/margins": -0.0054575116373598576,
+      "rewards/rejected": 0.06388974189758301,
+      "step": 620
+    },
+    {
+      "epoch": 0.504,
+      "grad_norm": 5.155455589294434,
+      "learning_rate": 8.315555555555556e-07,
+      "logits/chosen": 1.6201622486114502,
+      "logits/rejected": 1.6479911804199219,
+      "logps/chosen": -165.98980712890625,
+      "logps/rejected": -145.71644592285156,
+      "loss": 0.6804090023040772,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0773121565580368,
+      "rewards/margins": 0.027506589889526367,
+      "rewards/rejected": 0.049805570393800735,
+      "step": 630
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 4.009693145751953,
+      "learning_rate": 8.271111111111111e-07,
+      "logits/chosen": 1.5530269145965576,
+      "logits/rejected": 1.5585509538650513,
+      "logps/chosen": -166.77560424804688,
+      "logps/rejected": -151.09249877929688,
+      "loss": 0.6879617691040039,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07863648235797882,
+      "rewards/margins": 0.014577758498489857,
+      "rewards/rejected": 0.06405872106552124,
+      "step": 640
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 4.472072601318359,
+      "learning_rate": 8.226666666666666e-07,
+      "logits/chosen": 1.662239670753479,
+      "logits/rejected": 1.6585584878921509,
+      "logps/chosen": -153.26776123046875,
+      "logps/rejected": -125.24166107177734,
+      "loss": 0.6882720470428467,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.09423185139894485,
+      "rewards/margins": 0.011949598789215088,
+      "rewards/rejected": 0.08228223770856857,
+      "step": 650
+    },
+    {
+      "epoch": 0.528,
+      "grad_norm": 5.073488712310791,
+      "learning_rate": 8.182222222222222e-07,
+      "logits/chosen": 1.6752973794937134,
+      "logits/rejected": 1.6020495891571045,
+      "logps/chosen": -150.0669708251953,
+      "logps/rejected": -131.1305694580078,
+      "loss": 0.6880992889404297,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07629784196615219,
+      "rewards/margins": 0.01174530852586031,
+      "rewards/rejected": 0.06455253064632416,
+      "step": 660
+    },
+    {
+      "epoch": 0.536,
+      "grad_norm": 5.938063621520996,
+      "learning_rate": 8.137777777777777e-07,
+      "logits/chosen": 1.7563416957855225,
+      "logits/rejected": 1.5739262104034424,
+      "logps/chosen": -165.046875,
+      "logps/rejected": -150.13104248046875,
+      "loss": 0.6939912796020508,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.09020708501338959,
+      "rewards/margins": 0.0006308574229478836,
+      "rewards/rejected": 0.08957622945308685,
+      "step": 670
+    },
+    {
+      "epoch": 0.544,
+      "grad_norm": 4.362247467041016,
+      "learning_rate": 8.093333333333333e-07,
+      "logits/chosen": 1.6460363864898682,
+      "logits/rejected": 1.6379966735839844,
+      "logps/chosen": -143.24754333496094,
+      "logps/rejected": -131.8529815673828,
+      "loss": 0.6835652351379394,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.10346712917089462,
+      "rewards/margins": 0.02079077437520027,
+      "rewards/rejected": 0.08267635107040405,
+      "step": 680
+    },
+    {
+      "epoch": 0.552,
+      "grad_norm": 4.323369026184082,
+      "learning_rate": 8.048888888888888e-07,
+      "logits/chosen": 1.5466216802597046,
+      "logits/rejected": 1.541775107383728,
+      "logps/chosen": -171.82138061523438,
+      "logps/rejected": -158.87603759765625,
+      "loss": 0.6892048358917237,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0915951356291771,
+      "rewards/margins": 0.009338131174445152,
+      "rewards/rejected": 0.08225701004266739,
+      "step": 690
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 5.28114128112793,
+      "learning_rate": 8.004444444444444e-07,
+      "logits/chosen": 1.6419496536254883,
+      "logits/rejected": 1.6641361713409424,
+      "logps/chosen": -158.20787048339844,
+      "logps/rejected": -136.3108367919922,
+      "loss": 0.6985964775085449,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0700095146894455,
+      "rewards/margins": -0.008601363748311996,
+      "rewards/rejected": 0.07861088216304779,
+      "step": 700
+    },
+    {
+      "epoch": 0.56,
+      "eval_logits/chosen": 1.5386524200439453,
+      "eval_logits/rejected": 1.5675796270370483,
+      "eval_logps/chosen": -152.39877319335938,
+      "eval_logps/rejected": -147.0768585205078,
+      "eval_loss": 0.6893304586410522,
+      "eval_rewards/accuracies": 0.5320000052452087,
+      "eval_rewards/chosen": 0.09303704649209976,
+      "eval_rewards/margins": 0.010439171455800533,
+      "eval_rewards/rejected": 0.08259786665439606,
+      "eval_runtime": 90.3103,
+      "eval_samples_per_second": 5.536,
+      "eval_steps_per_second": 2.768,
+      "step": 700
+    },
+    {
+      "epoch": 0.568,
+      "grad_norm": 4.867155075073242,
+      "learning_rate": 7.96e-07,
+      "logits/chosen": 1.7137393951416016,
+      "logits/rejected": 1.6643224954605103,
+      "logps/chosen": -147.054931640625,
+      "logps/rejected": -162.10067749023438,
+      "loss": 0.6891860008239746,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.11649386584758759,
+      "rewards/margins": 0.00985223613679409,
+      "rewards/rejected": 0.10664163529872894,
+      "step": 710
+    },
+    {
+      "epoch": 0.576,
+      "grad_norm": 4.687198638916016,
+      "learning_rate": 7.915555555555556e-07,
+      "logits/chosen": 1.686532974243164,
+      "logits/rejected": 1.7992823123931885,
+      "logps/chosen": -138.60238647460938,
+      "logps/rejected": -134.22702026367188,
+      "loss": 0.7006660461425781,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.08661777526140213,
+      "rewards/margins": -0.013613695278763771,
+      "rewards/rejected": 0.10023146867752075,
+      "step": 720
+    },
+    {
+      "epoch": 0.584,
+      "grad_norm": 4.63344669342041,
+      "learning_rate": 7.87111111111111e-07,
+      "logits/chosen": 1.781561255455017,
+      "logits/rejected": 1.7561432123184204,
+      "logps/chosen": -151.60018920898438,
+      "logps/rejected": -147.93264770507812,
+      "loss": 0.6958267688751221,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.10642895847558975,
+      "rewards/margins": -0.002830044599249959,
+      "rewards/rejected": 0.1092589944601059,
+      "step": 730
+    },
+    {
+      "epoch": 0.592,
+      "grad_norm": 4.5400800704956055,
+      "learning_rate": 7.826666666666666e-07,
+      "logits/chosen": 1.670771837234497,
+      "logits/rejected": 1.5866410732269287,
+      "logps/chosen": -155.36764526367188,
+      "logps/rejected": -132.60902404785156,
+      "loss": 0.6921723842620849,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0910472720861435,
+      "rewards/margins": 0.003630922408774495,
+      "rewards/rejected": 0.08741635084152222,
+      "step": 740
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 4.779706954956055,
+      "learning_rate": 7.782222222222222e-07,
+      "logits/chosen": 1.531534194946289,
+      "logits/rejected": 1.5548356771469116,
+      "logps/chosen": -135.88177490234375,
+      "logps/rejected": -157.09231567382812,
+      "loss": 0.6919455528259277,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.09000497311353683,
+      "rewards/margins": 0.004116452299058437,
+      "rewards/rejected": 0.08588851988315582,
+      "step": 750
+    },
+    {
+      "epoch": 0.608,
+      "grad_norm": 5.283969879150391,
+      "learning_rate": 7.737777777777777e-07,
+      "logits/chosen": 1.6809686422348022,
+      "logits/rejected": 1.501511812210083,
+      "logps/chosen": -137.67315673828125,
+      "logps/rejected": -128.26022338867188,
+      "loss": 0.6907838344573974,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.08058954030275345,
+      "rewards/margins": 0.0064643076620996,
+      "rewards/rejected": 0.07412523031234741,
+      "step": 760
+    },
+    {
+      "epoch": 0.616,
+      "grad_norm": 4.341912269592285,
+      "learning_rate": 7.693333333333333e-07,
+      "logits/chosen": 1.684203863143921,
+      "logits/rejected": 1.6489808559417725,
+      "logps/chosen": -139.82455444335938,
+      "logps/rejected": -135.16998291015625,
+      "loss": 0.6793179988861084,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10346569865942001,
+      "rewards/margins": 0.02992106042802334,
+      "rewards/rejected": 0.07354463636875153,
+      "step": 770
+    },
+    {
+      "epoch": 0.624,
+      "grad_norm": 5.209469318389893,
+      "learning_rate": 7.648888888888888e-07,
+      "logits/chosen": 1.5599935054779053,
+      "logits/rejected": 1.6487398147583008,
+      "logps/chosen": -152.46170043945312,
+      "logps/rejected": -157.7329559326172,
+      "loss": 0.6873491287231446,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.10296590626239777,
+      "rewards/margins": 0.01368915755301714,
+      "rewards/rejected": 0.0892767459154129,
+      "step": 780
+    },
+    {
+      "epoch": 0.632,
+      "grad_norm": 5.364309310913086,
+      "learning_rate": 7.604444444444445e-07,
+      "logits/chosen": 1.5357733964920044,
+      "logits/rejected": 1.5833505392074585,
+      "logps/chosen": -146.4203338623047,
+      "logps/rejected": -149.77499389648438,
+      "loss": 0.68800368309021,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0752335637807846,
+      "rewards/margins": 0.01265893317759037,
+      "rewards/rejected": 0.06257463991641998,
+      "step": 790
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 4.701781272888184,
+      "learning_rate": 7.559999999999999e-07,
+      "logits/chosen": 1.7693378925323486,
+      "logits/rejected": 1.784106969833374,
+      "logps/chosen": -178.75717163085938,
+      "logps/rejected": -192.69229125976562,
+      "loss": 0.7001357078552246,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.07940518856048584,
+      "rewards/margins": -0.01138945110142231,
+      "rewards/rejected": 0.0907946228981018,
+      "step": 800
+    },
+    {
+      "epoch": 0.64,
+      "eval_logits/chosen": 1.532821536064148,
+      "eval_logits/rejected": 1.5615730285644531,
+      "eval_logps/chosen": -152.57040405273438,
+      "eval_logps/rejected": -147.24534606933594,
+      "eval_loss": 0.6894002556800842,
+      "eval_rewards/accuracies": 0.5339999794960022,
+      "eval_rewards/chosen": 0.07587439566850662,
+      "eval_rewards/margins": 0.010124183259904385,
+      "eval_rewards/rejected": 0.06575021147727966,
+      "eval_runtime": 90.2864,
+      "eval_samples_per_second": 5.538,
+      "eval_steps_per_second": 2.769,
+      "step": 800
+    },
+    {
+      "epoch": 0.648,
+      "grad_norm": 4.731827259063721,
+      "learning_rate": 7.515555555555555e-07,
+      "logits/chosen": 1.5014355182647705,
+      "logits/rejected": 1.706011176109314,
+      "logps/chosen": -113.23974609375,
+      "logps/rejected": -150.55316162109375,
+      "loss": 0.6896101951599121,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.066777303814888,
+      "rewards/margins": 0.008613836951553822,
+      "rewards/rejected": 0.05816347524523735,
+      "step": 810
+    },
+    {
+      "epoch": 0.656,
+      "grad_norm": 4.050163745880127,
+      "learning_rate": 7.47111111111111e-07,
+      "logits/chosen": 1.701898217201233,
+      "logits/rejected": 1.7274971008300781,
+      "logps/chosen": -147.45330810546875,
+      "logps/rejected": -140.33255004882812,
+      "loss": 0.6757836818695069,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08680602163076401,
+      "rewards/margins": 0.03675536438822746,
+      "rewards/rejected": 0.050050657242536545,
+      "step": 820
+    },
+    {
+      "epoch": 0.664,
+      "grad_norm": 4.168673992156982,
+      "learning_rate": 7.426666666666667e-07,
+      "logits/chosen": 1.6135514974594116,
+      "logits/rejected": 1.6518815755844116,
+      "logps/chosen": -137.38467407226562,
+      "logps/rejected": -132.65890502929688,
+      "loss": 0.6800778865814209,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.07235284894704819,
+      "rewards/margins": 0.028275374323129654,
+      "rewards/rejected": 0.04407747834920883,
+      "step": 830
+    },
+    {
+      "epoch": 0.672,
+      "grad_norm": 4.72458028793335,
+      "learning_rate": 7.382222222222222e-07,
+      "logits/chosen": 1.5987484455108643,
+      "logits/rejected": 1.6328668594360352,
+      "logps/chosen": -146.712158203125,
+      "logps/rejected": -156.0950469970703,
+      "loss": 0.6804145336151123,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.07264034450054169,
+      "rewards/margins": 0.0286283977329731,
+      "rewards/rejected": 0.04401194304227829,
+      "step": 840
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 4.735199928283691,
+      "learning_rate": 7.337777777777778e-07,
+      "logits/chosen": 1.6810247898101807,
+      "logits/rejected": 1.6662237644195557,
+      "logps/chosen": -159.40650939941406,
+      "logps/rejected": -140.65591430664062,
+      "loss": 0.6805107116699218,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.061445970088243484,
+      "rewards/margins": 0.02743927761912346,
+      "rewards/rejected": 0.034006692469120026,
+      "step": 850
+    },
+    {
+      "epoch": 0.688,
+      "grad_norm": 3.7038252353668213,
+      "learning_rate": 7.293333333333332e-07,
+      "logits/chosen": 1.6597106456756592,
+      "logits/rejected": 1.6951271295547485,
+      "logps/chosen": -138.1852569580078,
+      "logps/rejected": -128.6427764892578,
+      "loss": 0.6821750164031982,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.07160943746566772,
+      "rewards/margins": 0.023774990811944008,
+      "rewards/rejected": 0.047834448516368866,
+      "step": 860
+    },
+    {
+      "epoch": 0.696,
+      "grad_norm": 4.820807456970215,
+      "learning_rate": 7.248888888888888e-07,
+      "logits/chosen": 1.5708585977554321,
+      "logits/rejected": 1.5483477115631104,
+      "logps/chosen": -152.8867950439453,
+      "logps/rejected": -152.02584838867188,
+      "loss": 0.6911486625671387,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04338831081986427,
+      "rewards/margins": 0.006898392923176289,
+      "rewards/rejected": 0.03648992255330086,
+      "step": 870
+    },
+    {
+      "epoch": 0.704,
+      "grad_norm": 4.6849493980407715,
+      "learning_rate": 7.204444444444444e-07,
+      "logits/chosen": 1.5262442827224731,
+      "logits/rejected": 1.7751166820526123,
+      "logps/chosen": -143.77993774414062,
+      "logps/rejected": -155.7498016357422,
+      "loss": 0.6910871028900146,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.061417657881975174,
+      "rewards/margins": 0.006087464280426502,
+      "rewards/rejected": 0.055330194532871246,
+      "step": 880
+    },
+    {
+      "epoch": 0.712,
+      "grad_norm": 4.951540946960449,
+      "learning_rate": 7.159999999999999e-07,
+      "logits/chosen": 1.405790090560913,
+      "logits/rejected": 1.5980100631713867,
+      "logps/chosen": -147.6872100830078,
+      "logps/rejected": -160.23947143554688,
+      "loss": 0.6822467803955078,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.049768321216106415,
+      "rewards/margins": 0.024227874353528023,
+      "rewards/rejected": 0.02554045058786869,
+      "step": 890
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 4.255526542663574,
+      "learning_rate": 7.115555555555556e-07,
+      "logits/chosen": 1.6527436971664429,
+      "logits/rejected": 1.787755012512207,
+      "logps/chosen": -164.73355102539062,
+      "logps/rejected": -178.98507690429688,
+      "loss": 0.6861439704895019,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.07546674460172653,
+      "rewards/margins": 0.017860155552625656,
+      "rewards/rejected": 0.05760659649968147,
+      "step": 900
+    },
+    {
+      "epoch": 0.72,
+      "eval_logits/chosen": 1.5091361999511719,
+      "eval_logits/rejected": 1.5371856689453125,
+      "eval_logps/chosen": -152.8962860107422,
+      "eval_logps/rejected": -147.56655883789062,
+      "eval_loss": 0.690089225769043,
+      "eval_rewards/accuracies": 0.527999997138977,
+      "eval_rewards/chosen": 0.04328843951225281,
+      "eval_rewards/margins": 0.009660834446549416,
+      "eval_rewards/rejected": 0.03362761065363884,
+      "eval_runtime": 90.3227,
+      "eval_samples_per_second": 5.536,
+      "eval_steps_per_second": 2.768,
+      "step": 900
+    },
+    {
+      "epoch": 0.728,
+      "grad_norm": 4.453512668609619,
+      "learning_rate": 7.071111111111111e-07,
+      "logits/chosen": 1.700484037399292,
+      "logits/rejected": 1.4941186904907227,
+      "logps/chosen": -138.50682067871094,
+      "logps/rejected": -137.02490234375,
+      "loss": 0.6877517700195312,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05909284949302673,
+      "rewards/margins": 0.013610092923045158,
+      "rewards/rejected": 0.04548276215791702,
+      "step": 910
+    },
+    {
+      "epoch": 0.736,
+      "grad_norm": 5.548420429229736,
+      "learning_rate": 7.026666666666667e-07,
+      "logits/chosen": 1.409182071685791,
+      "logits/rejected": 1.375797152519226,
+      "logps/chosen": -158.7325439453125,
+      "logps/rejected": -161.9824981689453,
+      "loss": 0.6867617607116699,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.07141564786434174,
+      "rewards/margins": 0.015245514921844006,
+      "rewards/rejected": 0.05617012828588486,
+      "step": 920
+    },
+    {
+      "epoch": 0.744,
+      "grad_norm": 5.186211109161377,
+      "learning_rate": 6.982222222222221e-07,
+      "logits/chosen": 1.6255321502685547,
+      "logits/rejected": 1.7182047367095947,
+      "logps/chosen": -156.53213500976562,
+      "logps/rejected": -160.67556762695312,
+      "loss": 0.6812005519866944,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.062074560672044754,
+      "rewards/margins": 0.026883777230978012,
+      "rewards/rejected": 0.03519078344106674,
+      "step": 930
+    },
+    {
+      "epoch": 0.752,
+      "grad_norm": 4.044335842132568,
+      "learning_rate": 6.937777777777778e-07,
+      "logits/chosen": 1.6656415462493896,
+      "logits/rejected": 1.7865594625473022,
+      "logps/chosen": -147.00344848632812,
+      "logps/rejected": -173.11428833007812,
+      "loss": 0.690484619140625,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0269255880266428,
+      "rewards/margins": 0.007569611072540283,
+      "rewards/rejected": 0.019355975091457367,
+      "step": 940
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 4.8925299644470215,
+      "learning_rate": 6.893333333333333e-07,
+      "logits/chosen": 1.4856427907943726,
+      "logits/rejected": 1.5664136409759521,
+      "logps/chosen": -139.77938842773438,
+      "logps/rejected": -152.74557495117188,
+      "loss": 0.6817938804626464,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05857279896736145,
+      "rewards/margins": 0.02614629827439785,
+      "rewards/rejected": 0.03242649883031845,
+      "step": 950
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 4.514585018157959,
+      "learning_rate": 6.848888888888889e-07,
+      "logits/chosen": 1.6077378988265991,
+      "logits/rejected": 1.4770267009735107,
+      "logps/chosen": -134.602294921875,
+      "logps/rejected": -117.98567199707031,
+      "loss": 0.694350004196167,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.028881916776299477,
+      "rewards/margins": 0.00047348294174298644,
+      "rewards/rejected": 0.028408434242010117,
+      "step": 960
+    },
+    {
+      "epoch": 0.776,
+      "grad_norm": 3.9295125007629395,
+      "learning_rate": 6.804444444444444e-07,
+      "logits/chosen": 1.7124595642089844,
+      "logits/rejected": 1.8135782480239868,
+      "logps/chosen": -152.15426635742188,
+      "logps/rejected": -147.62945556640625,
+      "loss": 0.6929523944854736,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.057992029935121536,
+      "rewards/margins": 0.002844708738848567,
+      "rewards/rejected": 0.05514732003211975,
+      "step": 970
+    },
+    {
+      "epoch": 0.784,
+      "grad_norm": 5.510717391967773,
+      "learning_rate": 6.76e-07,
+      "logits/chosen": 1.483984351158142,
+      "logits/rejected": 1.4224226474761963,
+      "logps/chosen": -167.27133178710938,
+      "logps/rejected": -140.16891479492188,
+      "loss": 0.6842909336090088,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0873967856168747,
+      "rewards/margins": 0.020737329497933388,
+      "rewards/rejected": 0.06665945053100586,
+      "step": 980
+    },
+    {
+      "epoch": 0.792,
+      "grad_norm": 4.097748279571533,
+      "learning_rate": 6.715555555555556e-07,
+      "logits/chosen": 1.6006208658218384,
+      "logits/rejected": 1.6803340911865234,
+      "logps/chosen": -163.8089141845703,
+      "logps/rejected": -167.1127166748047,
+      "loss": 0.6833163261413574,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.07443337142467499,
+      "rewards/margins": 0.023210588842630386,
+      "rewards/rejected": 0.0512227788567543,
+      "step": 990
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 3.959730625152588,
+      "learning_rate": 6.67111111111111e-07,
+      "logits/chosen": 1.5806870460510254,
+      "logits/rejected": 1.5673385858535767,
+      "logps/chosen": -148.22584533691406,
+      "logps/rejected": -123.79376220703125,
+      "loss": 0.6892601490020752,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.07928630709648132,
+      "rewards/margins": 0.009630966000258923,
+      "rewards/rejected": 0.06965534389019012,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8,
+      "eval_logits/chosen": 1.522994875907898,
+      "eval_logits/rejected": 1.5511444807052612,
+      "eval_logps/chosen": -152.6211395263672,
+      "eval_logps/rejected": -147.31727600097656,
+      "eval_loss": 0.6889244914054871,
+      "eval_rewards/accuracies": 0.527999997138977,
+      "eval_rewards/chosen": 0.07080094516277313,
+      "eval_rewards/margins": 0.012245929799973965,
+      "eval_rewards/rejected": 0.058555010706186295,
+      "eval_runtime": 90.2821,
+      "eval_samples_per_second": 5.538,
+      "eval_steps_per_second": 2.769,
+      "step": 1000
+    },
+    {
+      "epoch": 0.808,
+      "grad_norm": 4.409601211547852,
+      "learning_rate": 6.626666666666666e-07,
+      "logits/chosen": 1.6469438076019287,
+      "logits/rejected": 1.6551824808120728,
+      "logps/chosen": -153.6597442626953,
+      "logps/rejected": -140.854248046875,
+      "loss": 0.6809319496154785,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.09710784256458282,
+      "rewards/margins": 0.026540305465459824,
+      "rewards/rejected": 0.0705675408244133,
+      "step": 1010
+    },
+    {
+      "epoch": 0.816,
+      "grad_norm": 6.559939384460449,
+      "learning_rate": 6.582222222222222e-07,
+      "logits/chosen": 1.5092687606811523,
+      "logits/rejected": 1.613526701927185,
+      "logps/chosen": -141.97103881835938,
+      "logps/rejected": -145.5518035888672,
+      "loss": 0.6865349292755127,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0591324046254158,
+      "rewards/margins": 0.017797131091356277,
+      "rewards/rejected": 0.041335273534059525,
+      "step": 1020
+    },
+    {
+      "epoch": 0.824,
+      "grad_norm": 5.1169047355651855,
+      "learning_rate": 6.537777777777778e-07,
+      "logits/chosen": 1.6573750972747803,
+      "logits/rejected": 1.4447792768478394,
+      "logps/chosen": -161.51220703125,
+      "logps/rejected": -135.18309020996094,
+      "loss": 0.6804659366607666,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10412336885929108,
+      "rewards/margins": 0.02856394089758396,
+      "rewards/rejected": 0.07555942982435226,
+      "step": 1030
+    },
+    {
+      "epoch": 0.832,
+      "grad_norm": 4.2392072677612305,
+      "learning_rate": 6.493333333333333e-07,
+      "logits/chosen": 1.6003319025039673,
+      "logits/rejected": 1.6346886157989502,
+      "logps/chosen": -139.08448791503906,
+      "logps/rejected": -139.89825439453125,
+      "loss": 0.6737568378448486,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.11095432937145233,
+      "rewards/margins": 0.043066851794719696,
+      "rewards/rejected": 0.06788748502731323,
+      "step": 1040
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 5.133569240570068,
+      "learning_rate": 6.448888888888889e-07,
+      "logits/chosen": 1.4209873676300049,
+      "logits/rejected": 1.5513734817504883,
+      "logps/chosen": -141.1917266845703,
+      "logps/rejected": -130.70431518554688,
+      "loss": 0.6861515998840332,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10248366743326187,
+      "rewards/margins": 0.016522446647286415,
+      "rewards/rejected": 0.08596121519804001,
+      "step": 1050
+    },
+    {
+      "epoch": 0.848,
+      "grad_norm": 4.0574445724487305,
+      "learning_rate": 6.404444444444444e-07,
+      "logits/chosen": 1.439145803451538,
+      "logits/rejected": 1.5661519765853882,
+      "logps/chosen": -111.7681884765625,
+      "logps/rejected": -126.37353515625,
+      "loss": 0.6691905975341796,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.15403084456920624,
+      "rewards/margins": 0.05248977616429329,
+      "rewards/rejected": 0.10154107958078384,
+      "step": 1060
+    },
+    {
+      "epoch": 0.856,
+      "grad_norm": 4.832082748413086,
+      "learning_rate": 6.36e-07,
+      "logits/chosen": 1.4902942180633545,
+      "logits/rejected": 1.3948299884796143,
+      "logps/chosen": -153.6760711669922,
+      "logps/rejected": -147.40023803710938,
+      "loss": 0.6729560375213623,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1259043663740158,
+      "rewards/margins": 0.0469796285033226,
+      "rewards/rejected": 0.0789247453212738,
+      "step": 1070
+    },
+    {
+      "epoch": 0.864,
+      "grad_norm": 3.609558343887329,
+      "learning_rate": 6.315555555555555e-07,
+      "logits/chosen": 1.5476783514022827,
+      "logits/rejected": 1.6365705728530884,
+      "logps/chosen": -148.82101440429688,
+      "logps/rejected": -122.3703842163086,
+      "loss": 0.6743530750274658,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1420261561870575,
+      "rewards/margins": 0.041455820202827454,
+      "rewards/rejected": 0.10057034343481064,
+      "step": 1080
+    },
+    {
+      "epoch": 0.872,
+      "grad_norm": 4.052758693695068,
+      "learning_rate": 6.27111111111111e-07,
+      "logits/chosen": 1.7934048175811768,
+      "logits/rejected": 1.7036349773406982,
+      "logps/chosen": -174.28765869140625,
+      "logps/rejected": -183.97897338867188,
+      "loss": 0.6748029708862304,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.1439160704612732,
+      "rewards/margins": 0.0408162847161293,
+      "rewards/rejected": 0.1030997782945633,
+      "step": 1090
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 5.463155269622803,
+      "learning_rate": 6.226666666666667e-07,
+      "logits/chosen": 1.5579731464385986,
+      "logits/rejected": 1.5956088304519653,
+      "logps/chosen": -138.96115112304688,
+      "logps/rejected": -152.51766967773438,
+      "loss": 0.6884016513824462,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.11226280778646469,
+      "rewards/margins": 0.013228577561676502,
+      "rewards/rejected": 0.09903421252965927,
+      "step": 1100
+    },
+    {
+      "epoch": 0.88,
+      "eval_logits/chosen": 1.5295906066894531,
+      "eval_logits/rejected": 1.5565518140792847,
+      "eval_logps/chosen": -152.15892028808594,
+      "eval_logps/rejected": -146.88125610351562,
+      "eval_loss": 0.6887561678886414,
+      "eval_rewards/accuracies": 0.5239999890327454,
+      "eval_rewards/chosen": 0.11702151596546173,
+      "eval_rewards/margins": 0.014860817231237888,
+      "eval_rewards/rejected": 0.10216069966554642,
+      "eval_runtime": 90.4847,
+      "eval_samples_per_second": 5.526,
+      "eval_steps_per_second": 2.763,
+      "step": 1100
+    },
+    {
+      "epoch": 0.888,
+      "grad_norm": 5.542585372924805,
+      "learning_rate": 6.182222222222222e-07,
+      "logits/chosen": 1.6245231628417969,
+      "logits/rejected": 1.600940465927124,
+      "logps/chosen": -155.20175170898438,
+      "logps/rejected": -144.58438110351562,
+      "loss": 0.6818556308746337,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.1249178797006607,
+      "rewards/margins": 0.027298670262098312,
+      "rewards/rejected": 0.0976191908121109,
+      "step": 1110
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 3.8583486080169678,
+      "learning_rate": 6.137777777777778e-07,
+      "logits/chosen": 1.6029832363128662,
+      "logits/rejected": 1.652834177017212,
+      "logps/chosen": -155.6839141845703,
+      "logps/rejected": -149.71646118164062,
+      "loss": 0.6816732883453369,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.13532081246376038,
+      "rewards/margins": 0.025635983794927597,
+      "rewards/rejected": 0.10968482494354248,
+      "step": 1120
+    },
+    {
+      "epoch": 0.904,
+      "grad_norm": 4.535235404968262,
+      "learning_rate": 6.093333333333332e-07,
+      "logits/chosen": 1.7116715908050537,
+      "logits/rejected": 1.5788238048553467,
+      "logps/chosen": -146.831787109375,
+      "logps/rejected": -134.20765686035156,
+      "loss": 0.6897575855255127,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.12892299890518188,
+      "rewards/margins": 0.011926446110010147,
+      "rewards/rejected": 0.11699654906988144,
+      "step": 1130
+    },
+    {
+      "epoch": 0.912,
+      "grad_norm": 4.356500148773193,
+      "learning_rate": 6.048888888888889e-07,
+      "logits/chosen": 1.6915569305419922,
+      "logits/rejected": 1.6864850521087646,
+      "logps/chosen": -154.83705139160156,
+      "logps/rejected": -141.92440795898438,
+      "loss": 0.677583646774292,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.13437099754810333,
+      "rewards/margins": 0.03629336506128311,
+      "rewards/rejected": 0.09807763993740082,
+      "step": 1140
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 4.3424553871154785,
+      "learning_rate": 6.004444444444444e-07,
+      "logits/chosen": 1.3488795757293701,
+      "logits/rejected": 1.3907456398010254,
+      "logps/chosen": -138.03089904785156,
+      "logps/rejected": -129.59719848632812,
+      "loss": 0.6962613582611084,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0907142236828804,
+      "rewards/margins": 0.0030510523356497288,
+      "rewards/rejected": 0.08766315877437592,
+      "step": 1150
+    },
+    {
+      "epoch": 0.928,
+      "grad_norm": 4.354366779327393,
+      "learning_rate": 5.96e-07,
+      "logits/chosen": 1.6015634536743164,
+      "logits/rejected": 1.5302627086639404,
+      "logps/chosen": -125.86180114746094,
+      "logps/rejected": -113.89128112792969,
+      "loss": 0.6872058868408203,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.11558832228183746,
+      "rewards/margins": 0.016637511551380157,
+      "rewards/rejected": 0.09895080327987671,
+      "step": 1160
+    },
+    {
+      "epoch": 0.936,
+      "grad_norm": 6.197093486785889,
+      "learning_rate": 5.915555555555555e-07,
+      "logits/chosen": 1.8193966150283813,
+      "logits/rejected": 1.7454732656478882,
+      "logps/chosen": -155.8551788330078,
+      "logps/rejected": -165.95828247070312,
+      "loss": 0.6903214454650879,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.12587358057498932,
+      "rewards/margins": 0.01157000008970499,
+      "rewards/rejected": 0.1143035739660263,
+      "step": 1170
+    },
+    {
+      "epoch": 0.944,
+      "grad_norm": 6.9796624183654785,
+      "learning_rate": 5.871111111111112e-07,
+      "logits/chosen": 1.7167119979858398,
+      "logits/rejected": 1.693549394607544,
+      "logps/chosen": -185.20008850097656,
+      "logps/rejected": -150.20620727539062,
+      "loss": 0.6912973880767822,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.1253143846988678,
+      "rewards/margins": 0.009792610071599483,
+      "rewards/rejected": 0.11552176624536514,
+      "step": 1180
+    },
+    {
+      "epoch": 0.952,
+      "grad_norm": 4.833356857299805,
+      "learning_rate": 5.826666666666666e-07,
+      "logits/chosen": 1.7434288263320923,
+      "logits/rejected": 1.7330595254898071,
+      "logps/chosen": -165.2196044921875,
+      "logps/rejected": -195.1993408203125,
+      "loss": 0.6812029361724854,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.11465537548065186,
+      "rewards/margins": 0.028904888778924942,
+      "rewards/rejected": 0.08575049787759781,
+      "step": 1190
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 4.727373123168945,
+      "learning_rate": 5.782222222222221e-07,
+      "logits/chosen": 1.6402454376220703,
+      "logits/rejected": 1.6071112155914307,
+      "logps/chosen": -138.0218963623047,
+      "logps/rejected": -144.56817626953125,
+      "loss": 0.694274616241455,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.11662209033966064,
+      "rewards/margins": 0.003029861254617572,
+      "rewards/rejected": 0.1135922223329544,
+      "step": 1200
+    },
+    {
+      "epoch": 0.96,
+      "eval_logits/chosen": 1.5258080959320068,
+      "eval_logits/rejected": 1.5528327226638794,
+      "eval_logps/chosen": -152.21652221679688,
+      "eval_logps/rejected": -146.95628356933594,
+      "eval_loss": 0.6881142854690552,
+      "eval_rewards/accuracies": 0.5239999890327454,
+      "eval_rewards/chosen": 0.11126487702131271,
+      "eval_rewards/margins": 0.01660888083279133,
+      "eval_rewards/rejected": 0.09465599805116653,
+      "eval_runtime": 90.2009,
+      "eval_samples_per_second": 5.543,
+      "eval_steps_per_second": 2.772,
+      "step": 1200
+    },
+    {
+      "epoch": 0.968,
+      "grad_norm": 4.669800758361816,
+      "learning_rate": 5.737777777777778e-07,
+      "logits/chosen": 1.469012975692749,
+      "logits/rejected": 1.4835999011993408,
+      "logps/chosen": -153.94541931152344,
+      "logps/rejected": -140.69659423828125,
+      "loss": 0.6921857357025146,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.12547791004180908,
+      "rewards/margins": 0.005948380567133427,
+      "rewards/rejected": 0.11952953040599823,
+      "step": 1210
+    },
+    {
+      "epoch": 0.976,
+      "grad_norm": 5.349202632904053,
+      "learning_rate": 5.693333333333333e-07,
+      "logits/chosen": 1.5056023597717285,
+      "logits/rejected": 1.4966309070587158,
+      "logps/chosen": -144.36300659179688,
+      "logps/rejected": -122.79240417480469,
+      "loss": 0.695373821258545,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.09502485394477844,
+      "rewards/margins": -0.0005314469453878701,
+      "rewards/rejected": 0.09555630385875702,
+      "step": 1220
+    },
+    {
+      "epoch": 0.984,
+      "grad_norm": 5.629171848297119,
+      "learning_rate": 5.648888888888889e-07,
+      "logits/chosen": 1.5664876699447632,
+      "logits/rejected": 1.7090803384780884,
+      "logps/chosen": -136.5504150390625,
+      "logps/rejected": -148.81802368164062,
+      "loss": 0.6888413906097413,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.13565854728221893,
+      "rewards/margins": 0.013341712765395641,
+      "rewards/rejected": 0.12231683731079102,
+      "step": 1230
+    },
+    {
+      "epoch": 0.992,
+      "grad_norm": 5.568618297576904,
+      "learning_rate": 5.604444444444444e-07,
+      "logits/chosen": 1.512286901473999,
+      "logits/rejected": 1.7082159519195557,
+      "logps/chosen": -143.9463653564453,
+      "logps/rejected": -146.82052612304688,
+      "loss": 0.6722752571105957,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.12829402089118958,
+      "rewards/margins": 0.046781741082668304,
+      "rewards/rejected": 0.08151227235794067,
+      "step": 1240
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 4.786505699157715,
+      "learning_rate": 5.560000000000001e-07,
+      "logits/chosen": 1.6615822315216064,
+      "logits/rejected": 1.7368179559707642,
+      "logps/chosen": -144.40292358398438,
+      "logps/rejected": -157.58763122558594,
+      "loss": 0.6829993724822998,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.11200229823589325,
+      "rewards/margins": 0.026959875598549843,
+      "rewards/rejected": 0.08504240959882736,
+      "step": 1250
+    },
+    {
+      "epoch": 1.008,
+      "grad_norm": 4.871355056762695,
+      "learning_rate": 5.515555555555555e-07,
+      "logits/chosen": 1.581756830215454,
+      "logits/rejected": 1.643133521080017,
+      "logps/chosen": -123.44065856933594,
+      "logps/rejected": -131.16763305664062,
+      "loss": 0.6869094848632813,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.11280514299869537,
+      "rewards/margins": 0.016064399853348732,
+      "rewards/rejected": 0.09674074500799179,
+      "step": 1260
+    },
+    {
+      "epoch": 1.016,
+      "grad_norm": 5.43352746963501,
+      "learning_rate": 5.471111111111111e-07,
+      "logits/chosen": 1.8056955337524414,
+      "logits/rejected": 1.7294971942901611,
+      "logps/chosen": -159.89236450195312,
+      "logps/rejected": -152.61293029785156,
+      "loss": 0.6701028347015381,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1585489809513092,
+      "rewards/margins": 0.0533718541264534,
+      "rewards/rejected": 0.1051771491765976,
+      "step": 1270
+    },
+    {
+      "epoch": 1.024,
+      "grad_norm": 5.0044474601745605,
+      "learning_rate": 5.426666666666666e-07,
+      "logits/chosen": 1.68048894405365,
+      "logits/rejected": 1.6845731735229492,
+      "logps/chosen": -153.21994018554688,
+      "logps/rejected": -140.06024169921875,
+      "loss": 0.675870132446289,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.11108388751745224,
+      "rewards/margins": 0.039517562836408615,
+      "rewards/rejected": 0.07156632840633392,
+      "step": 1280
+    },
+    {
+      "epoch": 1.032,
+      "grad_norm": 4.051770210266113,
+      "learning_rate": 5.382222222222223e-07,
+      "logits/chosen": 1.6341949701309204,
+      "logits/rejected": 1.7052574157714844,
+      "logps/chosen": -143.89512634277344,
+      "logps/rejected": -132.5374298095703,
+      "loss": 0.6808773040771484,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.11111323535442352,
+      "rewards/margins": 0.02908928692340851,
+      "rewards/rejected": 0.08202396333217621,
+      "step": 1290
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 4.0852885246276855,
+      "learning_rate": 5.337777777777778e-07,
+      "logits/chosen": 1.701703429222107,
+      "logits/rejected": 1.5303716659545898,
+      "logps/chosen": -149.3526611328125,
+      "logps/rejected": -129.2085418701172,
+      "loss": 0.6718564987182617,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.13976502418518066,
+      "rewards/margins": 0.04783231392502785,
+      "rewards/rejected": 0.09193271398544312,
+      "step": 1300
+    },
+    {
+      "epoch": 1.04,
+      "eval_logits/chosen": 1.5326013565063477,
+      "eval_logits/rejected": 1.5598564147949219,
+      "eval_logps/chosen": -152.15328979492188,
+      "eval_logps/rejected": -146.90542602539062,
+      "eval_loss": 0.6874103546142578,
+      "eval_rewards/accuracies": 0.5460000038146973,
+      "eval_rewards/chosen": 0.1175844818353653,
+      "eval_rewards/margins": 0.01784202829003334,
+      "eval_rewards/rejected": 0.09974244982004166,
+      "eval_runtime": 90.5137,
+      "eval_samples_per_second": 5.524,
+      "eval_steps_per_second": 2.762,
+      "step": 1300
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 2500,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

v5/DPO/DPO_10k/lora/checkpoint-1300/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677b288b67816c6ab7a9dcdd40d26bcb142fa3ad3ad050eaeeb4b73a1ba4b498
+size 6161

v5/DPO/DPO_10k/lora/checkpoint-2400/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v5/DPO/DPO_10k/lora/checkpoint-2400/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v5/DPO/DPO_10k/lora/checkpoint-2400/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:925fb02b11fdf9941bdfae0889066e9dab940f09395d0149a16abe025b37ad15
+size 180385008

v5/DPO/DPO_10k/lora/checkpoint-2400/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v5/DPO/DPO_10k/lora/checkpoint-2400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98ac224d5ed27ace528190a6990080c745116a4b920d737d850dcbf7cec83638
+size 360902475

v5/DPO/DPO_10k/lora/checkpoint-2400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7affab63b271ed0f59a5b53056fc0a581226a41dcdf2fc2b80b669e7c3cf714
+size 14645

v5/DPO/DPO_10k/lora/checkpoint-2400/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ee3be4931d9e7c32cc76f610ed77ea9a83d47b5639a53431c51dcc937f80ff5
+size 1383

v5/DPO/DPO_10k/lora/checkpoint-2400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:050d6987b00cc915a861681f6abd4fe1d6f88b067e04461ce21e4f85653a111f
+size 1465

v5/DPO/DPO_10k/lora/checkpoint-2400/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v5/DPO/DPO_10k/lora/checkpoint-2400/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v5/DPO/DPO_10k/lora/checkpoint-2400/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677b288b67816c6ab7a9dcdd40d26bcb142fa3ad3ad050eaeeb4b73a1ba4b498
+size 6161

v5/DPO/DPO_10k/lora/checkpoint-2500/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v5/DPO/DPO_10k/lora/checkpoint-2500/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "down_proj",
+    "k_proj",
+    "gate_proj",
+    "up_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v5/DPO/DPO_10k/lora/checkpoint-2500/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0bbd3c3e4c00bee837cf13980be86026bc37eed7182fa43a7b6a34481dc706f
+size 180385008

v5/DPO/DPO_10k/lora/checkpoint-2500/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

v5/DPO/DPO_10k/lora/checkpoint-2500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85a16bd9a9e3b84f02ff23b0aae6c239155a06a501d1d776ea15eba57fc952a
+size 360902475

v5/DPO/DPO_10k/lora/checkpoint-2500/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d1786ad2057a678cc204dadc7fc5d1a4f939be477df219f770c7d40e9270281
+size 14645

v5/DPO/DPO_10k/lora/checkpoint-2500/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4f7df58dbec84710c280fe8ef7660bf055a3d051df5a58eccd60f22b169583
+size 1383

v5/DPO/DPO_10k/lora/checkpoint-2500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7154cc492a37bc9bb0e6d09325e786c916142f1f119ecd183a60eecde4f9f283
+size 1465

v5/DPO/DPO_10k/lora/checkpoint-2500/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

v5/DPO/DPO_10k/lora/checkpoint-2500/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

v5/DPO/DPO_10k/lora/checkpoint-2500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:677b288b67816c6ab7a9dcdd40d26bcb142fa3ad3ad050eaeeb4b73a1ba4b498
+size 6161

v5/DPO/DPO_1k/DPO_1k/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-1B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-1B-Instruct
+- dpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

v5/DPO/DPO_1k/DPO_1k/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-1B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "up_proj",
+    "q_proj",
+    "down_proj",
+    "o_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

v5/DPO/DPO_1k/DPO_1k/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd39ce893511c42bb66fcc45902600954d24ae57a8646fa31ae8e964732c757a
+size 180385008