Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-200/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-363/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,72 @@
+---
+base_model: meta-llama/Llama-3.2-3B-Instruct
+library_name: peft
+model_name: simpo-xsum-low-mid-high-0220_0518
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-3B-Instruct
+- cpo
+- lora
+- transformers
+- trl
+licence: license
+pipeline_tag: text-generation
+---
+
+# Model Card for simpo-xsum-low-mid-high-0220_0518
+
+This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+This model was trained with CPO, a method introduced in [Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation](https://huggingface.co/papers/2401.08417).
+
+### Framework versions
+
+- PEFT 0.18.1
+- TRL: 0.28.0
+- Transformers: 5.2.0
+- Pytorch: 2.10.0+cu126
+- Datasets: 4.5.0
+- Tokenizers: 0.22.2
+
+## Citations
+
+Cite CPO as:
+
+```bibtex
+@inproceedings{xu2024contrastive,
+    title        = {{Contrastive Preference Optimization: Pushing the Boundaries of LLM Performance in Machine Translation}},
+    author       = {Haoran Xu and Amr Sharaf and Yunmo Chen and Weiting Tan and Lingfeng Shen and Benjamin Van Durme and Kenton Murray and Young Jin Kim},
+    year         = 2024,
+    booktitle    = {Forty-first International Conference on Machine Learning, {ICML} 2024, Vienna, Austria, July 21-27, 2024},
+    publisher    = {OpenReview.net},
+    url          = {https://openreview.net/forum?id=51iwkioZpn}
+}
+```
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b6a7a1799949e19405ea4a80e1bf4bc530d9447cdffc470d3860350906ed52
+size 97307544

chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-200/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-3B-Instruct
+- cpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-200/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-200/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91b6a7a1799949e19405ea4a80e1bf4bc530d9447cdffc470d3860350906ed52
+size 97307544

checkpoint-200/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48c10040a2dc536ed94a855ffed6c35ae910add4c1f2f3c77d4a8c977d212f9e
+size 194846331

checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a9f217e852f439efa6bd32fde98d6867f11aa6ea13ddc021ba10af6a0b0934
+size 14645

checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2aac6971ca7b4125148bea59a564076d6e29fe2e213fc111d8f6e073bcc84d3f
+size 1465

checkpoint-200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-200/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

checkpoint-200/trainer_state.json

@@ -0,0 +1,388 @@
+{
+  "best_global_step": 200,
+  "best_metric": 0.055398128926754,
+  "best_model_checkpoint": "/workspace/ckpts/simpo-xsum-low-mid-high-0220_0518/checkpoint-200",
+  "epoch": 0.5509641873278237,
+  "eval_steps": 100,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.027548209366391185,
+      "grad_norm": 0.700347900390625,
+      "learning_rate": 9e-06,
+      "logits/chosen": 0.42375749349594116,
+      "logits/rejected": 0.15861473977565765,
+      "logps/chosen": -3.2552437782287598,
+      "logps/rejected": -2.853081464767456,
+      "loss": 1.711391067504883,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -6.5104875564575195,
+      "rewards/margins": -0.8043240308761597,
+      "rewards/rejected": -5.706162929534912,
+      "step": 10
+    },
+    {
+      "epoch": 0.05509641873278237,
+      "grad_norm": 0.6906276345252991,
+      "learning_rate": 1.9e-05,
+      "logits/chosen": 0.4050915241241455,
+      "logits/rejected": 0.14284154772758484,
+      "logps/chosen": -2.8556487560272217,
+      "logps/rejected": -2.5371289253234863,
+      "loss": 1.5473010063171386,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -5.711297512054443,
+      "rewards/margins": -0.6370400190353394,
+      "rewards/rejected": -5.074257850646973,
+      "step": 20
+    },
+    {
+      "epoch": 0.08264462809917356,
+      "grad_norm": 0.5033866167068481,
+      "learning_rate": 2.9e-05,
+      "logits/chosen": 0.4237564206123352,
+      "logits/rejected": 0.19409556686878204,
+      "logps/chosen": -2.314438581466675,
+      "logps/rejected": -2.211418390274048,
+      "loss": 1.250732707977295,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -4.62887716293335,
+      "rewards/margins": -0.20604053139686584,
+      "rewards/rejected": -4.422836780548096,
+      "step": 30
+    },
+    {
+      "epoch": 0.11019283746556474,
+      "grad_norm": 0.5612661838531494,
+      "learning_rate": 3.9000000000000006e-05,
+      "logits/chosen": 0.1684650182723999,
+      "logits/rejected": -0.03625474497675896,
+      "logps/chosen": -1.9959722757339478,
+      "logps/rejected": -2.0295603275299072,
+      "loss": 1.053350067138672,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -3.9919445514678955,
+      "rewards/margins": 0.06717614084482193,
+      "rewards/rejected": -4.0591206550598145,
+      "step": 40
+    },
+    {
+      "epoch": 0.13774104683195593,
+      "grad_norm": 0.6303589940071106,
+      "learning_rate": 4.9e-05,
+      "logits/chosen": 0.23949162662029266,
+      "logits/rejected": 0.07211676985025406,
+      "logps/chosen": -1.8928591012954712,
+      "logps/rejected": -2.2719273567199707,
+      "loss": 0.6624780654907226,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -3.7857182025909424,
+      "rewards/margins": 0.7581357955932617,
+      "rewards/rejected": -4.543854713439941,
+      "step": 50
+    },
+    {
+      "epoch": 0.1652892561983471,
+      "grad_norm": 0.35993871092796326,
+      "learning_rate": 4.856230031948882e-05,
+      "logits/chosen": 0.5467869639396667,
+      "logits/rejected": 0.279127836227417,
+      "logps/chosen": -4.613501071929932,
+      "logps/rejected": -6.5180158615112305,
+      "loss": 0.24324963092803956,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -9.227002143859863,
+      "rewards/margins": 3.8090293407440186,
+      "rewards/rejected": -13.036031723022461,
+      "step": 60
+    },
+    {
+      "epoch": 0.1928374655647383,
+      "grad_norm": 1.6727081537246704,
+      "learning_rate": 4.696485623003195e-05,
+      "logits/chosen": 1.2077550888061523,
+      "logits/rejected": 1.0736474990844727,
+      "logps/chosen": -6.114351272583008,
+      "logps/rejected": -10.010709762573242,
+      "loss": 0.2190621852874756,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.228702545166016,
+      "rewards/margins": 7.792716979980469,
+      "rewards/rejected": -20.021419525146484,
+      "step": 70
+    },
+    {
+      "epoch": 0.22038567493112948,
+      "grad_norm": 0.6117808818817139,
+      "learning_rate": 4.536741214057508e-05,
+      "logits/chosen": 2.431880235671997,
+      "logits/rejected": 2.384572744369507,
+      "logps/chosen": -12.877838134765625,
+      "logps/rejected": -18.886180877685547,
+      "loss": 0.09992958307266235,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -25.75567626953125,
+      "rewards/margins": 12.016683578491211,
+      "rewards/rejected": -37.772361755371094,
+      "step": 80
+    },
+    {
+      "epoch": 0.24793388429752067,
+      "grad_norm": 0.7680425047874451,
+      "learning_rate": 4.376996805111822e-05,
+      "logits/chosen": 2.9793498516082764,
+      "logits/rejected": 2.9862256050109863,
+      "logps/chosen": -12.75178337097168,
+      "logps/rejected": -19.746002197265625,
+      "loss": 0.03834239840507507,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -25.50356674194336,
+      "rewards/margins": 13.988436698913574,
+      "rewards/rejected": -39.49200439453125,
+      "step": 90
+    },
+    {
+      "epoch": 0.27548209366391185,
+      "grad_norm": 0.06667827814817429,
+      "learning_rate": 4.217252396166134e-05,
+      "logits/chosen": 3.7260806560516357,
+      "logits/rejected": 3.7755050659179688,
+      "logps/chosen": -11.765376091003418,
+      "logps/rejected": -19.428905487060547,
+      "loss": 0.02194259166717529,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -23.530752182006836,
+      "rewards/margins": 15.327054023742676,
+      "rewards/rejected": -38.857810974121094,
+      "step": 100
+    },
+    {
+      "epoch": 0.27548209366391185,
+      "eval_logits/chosen": 5.057583808898926,
+      "eval_logits/rejected": 5.168581485748291,
+      "eval_logps/chosen": -17.074237823486328,
+      "eval_logps/rejected": -24.14946937561035,
+      "eval_loss": 0.0956403836607933,
+      "eval_nll_loss": 0.0,
+      "eval_rewards/accuracies": 0.9660714268684387,
+      "eval_rewards/chosen": -34.148475646972656,
+      "eval_rewards/margins": 14.150466918945312,
+      "eval_rewards/rejected": -48.2989387512207,
+      "eval_runtime": 40.0379,
+      "eval_samples_per_second": 13.912,
+      "eval_steps_per_second": 1.748,
+      "step": 100
+    },
+    {
+      "epoch": 0.30303030303030304,
+      "grad_norm": 0.7173494696617126,
+      "learning_rate": 4.0575079872204476e-05,
+      "logits/chosen": 3.5066864490509033,
+      "logits/rejected": 3.5185413360595703,
+      "logps/chosen": -9.856405258178711,
+      "logps/rejected": -16.73386001586914,
+      "loss": 0.12056845426559448,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -19.712810516357422,
+      "rewards/margins": 13.754910469055176,
+      "rewards/rejected": -33.46772003173828,
+      "step": 110
+    },
+    {
+      "epoch": 0.3305785123966942,
+      "grad_norm": 1.3427457809448242,
+      "learning_rate": 3.8977635782747605e-05,
+      "logits/chosen": 2.0150222778320312,
+      "logits/rejected": 2.052603244781494,
+      "logps/chosen": -6.702250003814697,
+      "logps/rejected": -12.878804206848145,
+      "loss": 0.05156264305114746,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -13.404500007629395,
+      "rewards/margins": 12.353109359741211,
+      "rewards/rejected": -25.75760841369629,
+      "step": 120
+    },
+    {
+      "epoch": 0.3581267217630854,
+      "grad_norm": 3.5135276317596436,
+      "learning_rate": 3.738019169329074e-05,
+      "logits/chosen": 2.870805263519287,
+      "logits/rejected": 3.019796371459961,
+      "logps/chosen": -10.168031692504883,
+      "logps/rejected": -17.399675369262695,
+      "loss": 0.05100045800209045,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -20.336063385009766,
+      "rewards/margins": 14.463289260864258,
+      "rewards/rejected": -34.79935073852539,
+      "step": 130
+    },
+    {
+      "epoch": 0.3856749311294766,
+      "grad_norm": 0.5818735957145691,
+      "learning_rate": 3.5782747603833865e-05,
+      "logits/chosen": 3.0271811485290527,
+      "logits/rejected": 3.1355438232421875,
+      "logps/chosen": -10.393911361694336,
+      "logps/rejected": -18.173566818237305,
+      "loss": 0.06783108711242676,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -20.787822723388672,
+      "rewards/margins": 15.55931282043457,
+      "rewards/rejected": -36.34713363647461,
+      "step": 140
+    },
+    {
+      "epoch": 0.4132231404958678,
+      "grad_norm": 7.5330939292907715,
+      "learning_rate": 3.4185303514377e-05,
+      "logits/chosen": 3.923037052154541,
+      "logits/rejected": 4.009919166564941,
+      "logps/chosen": -15.482580184936523,
+      "logps/rejected": -23.399499893188477,
+      "loss": 0.12233117818832398,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -30.965160369873047,
+      "rewards/margins": 15.833839416503906,
+      "rewards/rejected": -46.79899978637695,
+      "step": 150
+    },
+    {
+      "epoch": 0.44077134986225897,
+      "grad_norm": 0.0513874888420105,
+      "learning_rate": 3.258785942492013e-05,
+      "logits/chosen": 4.252791404724121,
+      "logits/rejected": 4.307219505310059,
+      "logps/chosen": -13.000323295593262,
+      "logps/rejected": -21.655231475830078,
+      "loss": 0.03463771939277649,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -26.000646591186523,
+      "rewards/margins": 17.309818267822266,
+      "rewards/rejected": -43.310462951660156,
+      "step": 160
+    },
+    {
+      "epoch": 0.46831955922865015,
+      "grad_norm": 9.692298772279173e-05,
+      "learning_rate": 3.099041533546326e-05,
+      "logits/chosen": 4.098459243774414,
+      "logits/rejected": 4.160243511199951,
+      "logps/chosen": -11.710687637329102,
+      "logps/rejected": -21.676971435546875,
+      "loss": 0.024555636942386626,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -23.421375274658203,
+      "rewards/margins": 19.932567596435547,
+      "rewards/rejected": -43.35394287109375,
+      "step": 170
+    },
+    {
+      "epoch": 0.49586776859504134,
+      "grad_norm": 1.611127495765686,
+      "learning_rate": 2.939297124600639e-05,
+      "logits/chosen": 4.176002025604248,
+      "logits/rejected": 4.312728404998779,
+      "logps/chosen": -13.934991836547852,
+      "logps/rejected": -24.690296173095703,
+      "loss": 0.02139030247926712,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -27.869983673095703,
+      "rewards/margins": 21.51060676574707,
+      "rewards/rejected": -49.380592346191406,
+      "step": 180
+    },
+    {
+      "epoch": 0.5234159779614325,
+      "grad_norm": 1.21311616897583,
+      "learning_rate": 2.7795527156549523e-05,
+      "logits/chosen": 3.4004039764404297,
+      "logits/rejected": 3.5335049629211426,
+      "logps/chosen": -9.917387008666992,
+      "logps/rejected": -21.205930709838867,
+      "loss": 0.028738826513290405,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -19.834774017333984,
+      "rewards/margins": 22.577091217041016,
+      "rewards/rejected": -42.411861419677734,
+      "step": 190
+    },
+    {
+      "epoch": 0.5509641873278237,
+      "grad_norm": 8.95163631439209,
+      "learning_rate": 2.6198083067092656e-05,
+      "logits/chosen": 3.074721336364746,
+      "logits/rejected": 3.1740307807922363,
+      "logps/chosen": -10.301236152648926,
+      "logps/rejected": -22.369190216064453,
+      "loss": 0.11196700334548951,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -20.60247230529785,
+      "rewards/margins": 24.135910034179688,
+      "rewards/rejected": -44.738380432128906,
+      "step": 200
+    },
+    {
+      "epoch": 0.5509641873278237,
+      "eval_logits/chosen": 3.9634761810302734,
+      "eval_logits/rejected": 4.095207691192627,
+      "eval_logps/chosen": -10.509795188903809,
+      "eval_logps/rejected": -22.200040817260742,
+      "eval_loss": 0.055398128926754,
+      "eval_nll_loss": 0.0,
+      "eval_rewards/accuracies": 0.9857142567634583,
+      "eval_rewards/chosen": -21.019590377807617,
+      "eval_rewards/margins": 23.380483627319336,
+      "eval_rewards/rejected": -44.400081634521484,
+      "eval_runtime": 40.0513,
+      "eval_samples_per_second": 13.907,
+      "eval_steps_per_second": 1.748,
+      "step": 200
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 363,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3bae6079ee85f272ee5c1148388807254b89f462d2cec3c58b92c2a1cb918d
+size 5585

checkpoint-363/README.md

@@ -0,0 +1,209 @@
+---
+base_model: meta-llama/Llama-3.2-3B-Instruct
+library_name: peft
+pipeline_tag: text-generation
+tags:
+- base_model:adapter:meta-llama/Llama-3.2-3B-Instruct
+- cpo
+- lora
+- transformers
+- trl
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.18.1

checkpoint-363/adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj",
+    "v_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

checkpoint-363/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ba300c39aa9407a9d2664880c936d5433dd8f22fb86115d09205c8e8d96818d
+size 97307544

checkpoint-363/chat_template.jinja

@@ -0,0 +1,93 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- if strftime_now is defined %}
+        {%- set date_string = strftime_now("%d %b %Y") %}
+    {%- else %}
+        {%- set date_string = "26 Jul 2024" %}
+    {%- endif %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+        {{- '{"name": "' + tool_call.name + '", ' }}
+        {{- '"parameters": ' }}
+        {{- tool_call.arguments | tojson }}
+        {{- "}" }}
+        {{- "<|eot_id|>" }}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

checkpoint-363/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8a9e245ea537c892b222285fb04eb5f408b6e146bd058f237fae6b7cdac7036
+size 194846331

checkpoint-363/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e2011629d8bed3ef560fa11175cac55684c4e12a72634bb24abf767b6c7399
+size 14645

checkpoint-363/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0c423b41cddc772a681bf0ccebe0fe1a1ad6cd26b18884ca76701b58d6f95ba
+size 1465

checkpoint-363/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-363/tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

checkpoint-363/trainer_state.json

@@ -0,0 +1,661 @@
+{
+  "best_global_step": 200,
+  "best_metric": 0.055398128926754,
+  "best_model_checkpoint": "/workspace/ckpts/simpo-xsum-low-mid-high-0220_0518/checkpoint-200",
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 363,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.027548209366391185,
+      "grad_norm": 0.700347900390625,
+      "learning_rate": 9e-06,
+      "logits/chosen": 0.42375749349594116,
+      "logits/rejected": 0.15861473977565765,
+      "logps/chosen": -3.2552437782287598,
+      "logps/rejected": -2.853081464767456,
+      "loss": 1.711391067504883,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.3125,
+      "rewards/chosen": -6.5104875564575195,
+      "rewards/margins": -0.8043240308761597,
+      "rewards/rejected": -5.706162929534912,
+      "step": 10
+    },
+    {
+      "epoch": 0.05509641873278237,
+      "grad_norm": 0.6906276345252991,
+      "learning_rate": 1.9e-05,
+      "logits/chosen": 0.4050915241241455,
+      "logits/rejected": 0.14284154772758484,
+      "logps/chosen": -2.8556487560272217,
+      "logps/rejected": -2.5371289253234863,
+      "loss": 1.5473010063171386,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -5.711297512054443,
+      "rewards/margins": -0.6370400190353394,
+      "rewards/rejected": -5.074257850646973,
+      "step": 20
+    },
+    {
+      "epoch": 0.08264462809917356,
+      "grad_norm": 0.5033866167068481,
+      "learning_rate": 2.9e-05,
+      "logits/chosen": 0.4237564206123352,
+      "logits/rejected": 0.19409556686878204,
+      "logps/chosen": -2.314438581466675,
+      "logps/rejected": -2.211418390274048,
+      "loss": 1.250732707977295,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -4.62887716293335,
+      "rewards/margins": -0.20604053139686584,
+      "rewards/rejected": -4.422836780548096,
+      "step": 30
+    },
+    {
+      "epoch": 0.11019283746556474,
+      "grad_norm": 0.5612661838531494,
+      "learning_rate": 3.9000000000000006e-05,
+      "logits/chosen": 0.1684650182723999,
+      "logits/rejected": -0.03625474497675896,
+      "logps/chosen": -1.9959722757339478,
+      "logps/rejected": -2.0295603275299072,
+      "loss": 1.053350067138672,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -3.9919445514678955,
+      "rewards/margins": 0.06717614084482193,
+      "rewards/rejected": -4.0591206550598145,
+      "step": 40
+    },
+    {
+      "epoch": 0.13774104683195593,
+      "grad_norm": 0.6303589940071106,
+      "learning_rate": 4.9e-05,
+      "logits/chosen": 0.23949162662029266,
+      "logits/rejected": 0.07211676985025406,
+      "logps/chosen": -1.8928591012954712,
+      "logps/rejected": -2.2719273567199707,
+      "loss": 0.6624780654907226,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -3.7857182025909424,
+      "rewards/margins": 0.7581357955932617,
+      "rewards/rejected": -4.543854713439941,
+      "step": 50
+    },
+    {
+      "epoch": 0.1652892561983471,
+      "grad_norm": 0.35993871092796326,
+      "learning_rate": 4.856230031948882e-05,
+      "logits/chosen": 0.5467869639396667,
+      "logits/rejected": 0.279127836227417,
+      "logps/chosen": -4.613501071929932,
+      "logps/rejected": -6.5180158615112305,
+      "loss": 0.24324963092803956,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -9.227002143859863,
+      "rewards/margins": 3.8090293407440186,
+      "rewards/rejected": -13.036031723022461,
+      "step": 60
+    },
+    {
+      "epoch": 0.1928374655647383,
+      "grad_norm": 1.6727081537246704,
+      "learning_rate": 4.696485623003195e-05,
+      "logits/chosen": 1.2077550888061523,
+      "logits/rejected": 1.0736474990844727,
+      "logps/chosen": -6.114351272583008,
+      "logps/rejected": -10.010709762573242,
+      "loss": 0.2190621852874756,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -12.228702545166016,
+      "rewards/margins": 7.792716979980469,
+      "rewards/rejected": -20.021419525146484,
+      "step": 70
+    },
+    {
+      "epoch": 0.22038567493112948,
+      "grad_norm": 0.6117808818817139,
+      "learning_rate": 4.536741214057508e-05,
+      "logits/chosen": 2.431880235671997,
+      "logits/rejected": 2.384572744369507,
+      "logps/chosen": -12.877838134765625,
+      "logps/rejected": -18.886180877685547,
+      "loss": 0.09992958307266235,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -25.75567626953125,
+      "rewards/margins": 12.016683578491211,
+      "rewards/rejected": -37.772361755371094,
+      "step": 80
+    },
+    {
+      "epoch": 0.24793388429752067,
+      "grad_norm": 0.7680425047874451,
+      "learning_rate": 4.376996805111822e-05,
+      "logits/chosen": 2.9793498516082764,
+      "logits/rejected": 2.9862256050109863,
+      "logps/chosen": -12.75178337097168,
+      "logps/rejected": -19.746002197265625,
+      "loss": 0.03834239840507507,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -25.50356674194336,
+      "rewards/margins": 13.988436698913574,
+      "rewards/rejected": -39.49200439453125,
+      "step": 90
+    },
+    {
+      "epoch": 0.27548209366391185,
+      "grad_norm": 0.06667827814817429,
+      "learning_rate": 4.217252396166134e-05,
+      "logits/chosen": 3.7260806560516357,
+      "logits/rejected": 3.7755050659179688,
+      "logps/chosen": -11.765376091003418,
+      "logps/rejected": -19.428905487060547,
+      "loss": 0.02194259166717529,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -23.530752182006836,
+      "rewards/margins": 15.327054023742676,
+      "rewards/rejected": -38.857810974121094,
+      "step": 100
+    },
+    {
+      "epoch": 0.27548209366391185,
+      "eval_logits/chosen": 5.057583808898926,
+      "eval_logits/rejected": 5.168581485748291,
+      "eval_logps/chosen": -17.074237823486328,
+      "eval_logps/rejected": -24.14946937561035,
+      "eval_loss": 0.0956403836607933,
+      "eval_nll_loss": 0.0,
+      "eval_rewards/accuracies": 0.9660714268684387,
+      "eval_rewards/chosen": -34.148475646972656,
+      "eval_rewards/margins": 14.150466918945312,
+      "eval_rewards/rejected": -48.2989387512207,
+      "eval_runtime": 40.0379,
+      "eval_samples_per_second": 13.912,
+      "eval_steps_per_second": 1.748,
+      "step": 100
+    },
+    {
+      "epoch": 0.30303030303030304,
+      "grad_norm": 0.7173494696617126,
+      "learning_rate": 4.0575079872204476e-05,
+      "logits/chosen": 3.5066864490509033,
+      "logits/rejected": 3.5185413360595703,
+      "logps/chosen": -9.856405258178711,
+      "logps/rejected": -16.73386001586914,
+      "loss": 0.12056845426559448,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -19.712810516357422,
+      "rewards/margins": 13.754910469055176,
+      "rewards/rejected": -33.46772003173828,
+      "step": 110
+    },
+    {
+      "epoch": 0.3305785123966942,
+      "grad_norm": 1.3427457809448242,
+      "learning_rate": 3.8977635782747605e-05,
+      "logits/chosen": 2.0150222778320312,
+      "logits/rejected": 2.052603244781494,
+      "logps/chosen": -6.702250003814697,
+      "logps/rejected": -12.878804206848145,
+      "loss": 0.05156264305114746,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -13.404500007629395,
+      "rewards/margins": 12.353109359741211,
+      "rewards/rejected": -25.75760841369629,
+      "step": 120
+    },
+    {
+      "epoch": 0.3581267217630854,
+      "grad_norm": 3.5135276317596436,
+      "learning_rate": 3.738019169329074e-05,
+      "logits/chosen": 2.870805263519287,
+      "logits/rejected": 3.019796371459961,
+      "logps/chosen": -10.168031692504883,
+      "logps/rejected": -17.399675369262695,
+      "loss": 0.05100045800209045,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -20.336063385009766,
+      "rewards/margins": 14.463289260864258,
+      "rewards/rejected": -34.79935073852539,
+      "step": 130
+    },
+    {
+      "epoch": 0.3856749311294766,
+      "grad_norm": 0.5818735957145691,
+      "learning_rate": 3.5782747603833865e-05,
+      "logits/chosen": 3.0271811485290527,
+      "logits/rejected": 3.1355438232421875,
+      "logps/chosen": -10.393911361694336,
+      "logps/rejected": -18.173566818237305,
+      "loss": 0.06783108711242676,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -20.787822723388672,
+      "rewards/margins": 15.55931282043457,
+      "rewards/rejected": -36.34713363647461,
+      "step": 140
+    },
+    {
+      "epoch": 0.4132231404958678,
+      "grad_norm": 7.5330939292907715,
+      "learning_rate": 3.4185303514377e-05,
+      "logits/chosen": 3.923037052154541,
+      "logits/rejected": 4.009919166564941,
+      "logps/chosen": -15.482580184936523,
+      "logps/rejected": -23.399499893188477,
+      "loss": 0.12233117818832398,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -30.965160369873047,
+      "rewards/margins": 15.833839416503906,
+      "rewards/rejected": -46.79899978637695,
+      "step": 150
+    },
+    {
+      "epoch": 0.44077134986225897,
+      "grad_norm": 0.0513874888420105,
+      "learning_rate": 3.258785942492013e-05,
+      "logits/chosen": 4.252791404724121,
+      "logits/rejected": 4.307219505310059,
+      "logps/chosen": -13.000323295593262,
+      "logps/rejected": -21.655231475830078,
+      "loss": 0.03463771939277649,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -26.000646591186523,
+      "rewards/margins": 17.309818267822266,
+      "rewards/rejected": -43.310462951660156,
+      "step": 160
+    },
+    {
+      "epoch": 0.46831955922865015,
+      "grad_norm": 9.692298772279173e-05,
+      "learning_rate": 3.099041533546326e-05,
+      "logits/chosen": 4.098459243774414,
+      "logits/rejected": 4.160243511199951,
+      "logps/chosen": -11.710687637329102,
+      "logps/rejected": -21.676971435546875,
+      "loss": 0.024555636942386626,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -23.421375274658203,
+      "rewards/margins": 19.932567596435547,
+      "rewards/rejected": -43.35394287109375,
+      "step": 170
+    },
+    {
+      "epoch": 0.49586776859504134,
+      "grad_norm": 1.611127495765686,
+      "learning_rate": 2.939297124600639e-05,
+      "logits/chosen": 4.176002025604248,
+      "logits/rejected": 4.312728404998779,
+      "logps/chosen": -13.934991836547852,
+      "logps/rejected": -24.690296173095703,
+      "loss": 0.02139030247926712,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -27.869983673095703,
+      "rewards/margins": 21.51060676574707,
+      "rewards/rejected": -49.380592346191406,
+      "step": 180
+    },
+    {
+      "epoch": 0.5234159779614325,
+      "grad_norm": 1.21311616897583,
+      "learning_rate": 2.7795527156549523e-05,
+      "logits/chosen": 3.4004039764404297,
+      "logits/rejected": 3.5335049629211426,
+      "logps/chosen": -9.917387008666992,
+      "logps/rejected": -21.205930709838867,
+      "loss": 0.028738826513290405,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -19.834774017333984,
+      "rewards/margins": 22.577091217041016,
+      "rewards/rejected": -42.411861419677734,
+      "step": 190
+    },
+    {
+      "epoch": 0.5509641873278237,
+      "grad_norm": 8.95163631439209,
+      "learning_rate": 2.6198083067092656e-05,
+      "logits/chosen": 3.074721336364746,
+      "logits/rejected": 3.1740307807922363,
+      "logps/chosen": -10.301236152648926,
+      "logps/rejected": -22.369190216064453,
+      "loss": 0.11196700334548951,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -20.60247230529785,
+      "rewards/margins": 24.135910034179688,
+      "rewards/rejected": -44.738380432128906,
+      "step": 200
+    },
+    {
+      "epoch": 0.5509641873278237,
+      "eval_logits/chosen": 3.9634761810302734,
+      "eval_logits/rejected": 4.095207691192627,
+      "eval_logps/chosen": -10.509795188903809,
+      "eval_logps/rejected": -22.200040817260742,
+      "eval_loss": 0.055398128926754,
+      "eval_nll_loss": 0.0,
+      "eval_rewards/accuracies": 0.9857142567634583,
+      "eval_rewards/chosen": -21.019590377807617,
+      "eval_rewards/margins": 23.380483627319336,
+      "eval_rewards/rejected": -44.400081634521484,
+      "eval_runtime": 40.0513,
+      "eval_samples_per_second": 13.907,
+      "eval_steps_per_second": 1.748,
+      "step": 200
+    },
+    {
+      "epoch": 0.5785123966942148,
+      "grad_norm": 2.871490955352783,
+      "learning_rate": 2.4600638977635785e-05,
+      "logits/chosen": 2.7474374771118164,
+      "logits/rejected": 2.8057074546813965,
+      "logps/chosen": -6.542741298675537,
+      "logps/rejected": -17.251813888549805,
+      "loss": 0.05973906517028808,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -13.085482597351074,
+      "rewards/margins": 21.418142318725586,
+      "rewards/rejected": -34.50362777709961,
+      "step": 210
+    },
+    {
+      "epoch": 0.6060606060606061,
+      "grad_norm": 0.013486926443874836,
+      "learning_rate": 2.3003194888178915e-05,
+      "logits/chosen": 2.420773506164551,
+      "logits/rejected": 2.4592440128326416,
+      "logps/chosen": -4.120817184448242,
+      "logps/rejected": -13.536539077758789,
+      "loss": 0.03502379059791565,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -8.241634368896484,
+      "rewards/margins": 18.831443786621094,
+      "rewards/rejected": -27.073078155517578,
+      "step": 220
+    },
+    {
+      "epoch": 0.6336088154269972,
+      "grad_norm": 4.076554775238037,
+      "learning_rate": 2.1405750798722048e-05,
+      "logits/chosen": 3.2621731758117676,
+      "logits/rejected": 3.4068379402160645,
+      "logps/chosen": -7.666436672210693,
+      "logps/rejected": -18.324886322021484,
+      "loss": 0.12405169010162354,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -15.332873344421387,
+      "rewards/margins": 21.316898345947266,
+      "rewards/rejected": -36.64977264404297,
+      "step": 230
+    },
+    {
+      "epoch": 0.6611570247933884,
+      "grad_norm": 1.421222448348999,
+      "learning_rate": 1.9808306709265177e-05,
+      "logits/chosen": 3.560548782348633,
+      "logits/rejected": 3.6851909160614014,
+      "logps/chosen": -9.624505996704102,
+      "logps/rejected": -20.099857330322266,
+      "loss": 0.03761735558509827,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -19.249011993408203,
+      "rewards/margins": 20.95070457458496,
+      "rewards/rejected": -40.19971466064453,
+      "step": 240
+    },
+    {
+      "epoch": 0.6887052341597796,
+      "grad_norm": 0.0010234045330435038,
+      "learning_rate": 1.8210862619808307e-05,
+      "logits/chosen": 3.57916259765625,
+      "logits/rejected": 3.6618869304656982,
+      "logps/chosen": -8.578134536743164,
+      "logps/rejected": -20.71200180053711,
+      "loss": 0.09314435720443726,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.156269073486328,
+      "rewards/margins": 24.26773452758789,
+      "rewards/rejected": -41.42400360107422,
+      "step": 250
+    },
+    {
+      "epoch": 0.7162534435261708,
+      "grad_norm": 0.12575837969779968,
+      "learning_rate": 1.6613418530351437e-05,
+      "logits/chosen": 3.0451531410217285,
+      "logits/rejected": 3.036803722381592,
+      "logps/chosen": -5.424054145812988,
+      "logps/rejected": -15.074081420898438,
+      "loss": 0.013192214071750641,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -10.848108291625977,
+      "rewards/margins": 19.3000545501709,
+      "rewards/rejected": -30.148162841796875,
+      "step": 260
+    },
+    {
+      "epoch": 0.743801652892562,
+      "grad_norm": 3.9852426052093506,
+      "learning_rate": 1.501597444089457e-05,
+      "logits/chosen": 3.3318965435028076,
+      "logits/rejected": 3.3611156940460205,
+      "logps/chosen": -6.45603084564209,
+      "logps/rejected": -16.13960075378418,
+      "loss": 0.1191794753074646,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -12.91206169128418,
+      "rewards/margins": 19.367137908935547,
+      "rewards/rejected": -32.27920150756836,
+      "step": 270
+    },
+    {
+      "epoch": 0.7713498622589532,
+      "grad_norm": 0.00017260252207051963,
+      "learning_rate": 1.34185303514377e-05,
+      "logits/chosen": 2.8985068798065186,
+      "logits/rejected": 2.9405500888824463,
+      "logps/chosen": -5.470853328704834,
+      "logps/rejected": -17.427152633666992,
+      "loss": 0.11677546501159668,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -10.941706657409668,
+      "rewards/margins": 23.912601470947266,
+      "rewards/rejected": -34.854305267333984,
+      "step": 280
+    },
+    {
+      "epoch": 0.7988980716253443,
+      "grad_norm": 1.2612056732177734,
+      "learning_rate": 1.182108626198083e-05,
+      "logits/chosen": 2.612856864929199,
+      "logits/rejected": 2.5626139640808105,
+      "logps/chosen": -4.082730770111084,
+      "logps/rejected": -15.125768661499023,
+      "loss": 0.06155251264572144,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -8.165461540222168,
+      "rewards/margins": 22.086074829101562,
+      "rewards/rejected": -30.251537322998047,
+      "step": 290
+    },
+    {
+      "epoch": 0.8264462809917356,
+      "grad_norm": 0.00040509909740649164,
+      "learning_rate": 1.0223642172523962e-05,
+      "logits/chosen": 2.4162771701812744,
+      "logits/rejected": 2.3998305797576904,
+      "logps/chosen": -3.854151487350464,
+      "logps/rejected": -13.961355209350586,
+      "loss": 0.024074266850948333,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -7.708302974700928,
+      "rewards/margins": 20.214406967163086,
+      "rewards/rejected": -27.922710418701172,
+      "step": 300
+    },
+    {
+      "epoch": 0.8264462809917356,
+      "eval_logits/chosen": 3.4136674404144287,
+      "eval_logits/rejected": 3.45212721824646,
+      "eval_logps/chosen": -4.449319362640381,
+      "eval_logps/rejected": -15.445338249206543,
+      "eval_loss": 0.0694827288389206,
+      "eval_nll_loss": 0.0,
+      "eval_rewards/accuracies": 0.9857142567634583,
+      "eval_rewards/chosen": -8.898638725280762,
+      "eval_rewards/margins": 21.992034912109375,
+      "eval_rewards/rejected": -30.890676498413086,
+      "eval_runtime": 40.0626,
+      "eval_samples_per_second": 13.903,
+      "eval_steps_per_second": 1.747,
+      "step": 300
+    },
+    {
+      "epoch": 0.8539944903581267,
+      "grad_norm": 1.521673560142517,
+      "learning_rate": 8.626198083067093e-06,
+      "logits/chosen": 2.45418381690979,
+      "logits/rejected": 2.533856153488159,
+      "logps/chosen": -4.843596458435059,
+      "logps/rejected": -17.073049545288086,
+      "loss": 0.08694828748703003,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -9.687192916870117,
+      "rewards/margins": 24.458908081054688,
+      "rewards/rejected": -34.14609909057617,
+      "step": 310
+    },
+    {
+      "epoch": 0.8815426997245179,
+      "grad_norm": 2.497852165106451e-07,
+      "learning_rate": 7.0287539936102235e-06,
+      "logits/chosen": 2.896289110183716,
+      "logits/rejected": 2.986301898956299,
+      "logps/chosen": -6.26573371887207,
+      "logps/rejected": -19.259273529052734,
+      "loss": 0.04625120460987091,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -12.53146743774414,
+      "rewards/margins": 25.98708152770996,
+      "rewards/rejected": -38.51854705810547,
+      "step": 320
+    },
+    {
+      "epoch": 0.9090909090909091,
+      "grad_norm": 0.003939513117074966,
+      "learning_rate": 5.431309904153355e-06,
+      "logits/chosen": 2.6858134269714355,
+      "logits/rejected": 2.763576030731201,
+      "logps/chosen": -6.823577880859375,
+      "logps/rejected": -17.725753784179688,
+      "loss": 0.043733158707618715,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -13.64715576171875,
+      "rewards/margins": 21.804353713989258,
+      "rewards/rejected": -35.451507568359375,
+      "step": 330
+    },
+    {
+      "epoch": 0.9366391184573003,
+      "grad_norm": 0.0016138068167492747,
+      "learning_rate": 3.833865814696485e-06,
+      "logits/chosen": 3.1492819786071777,
+      "logits/rejected": 3.2142040729522705,
+      "logps/chosen": -7.908546447753906,
+      "logps/rejected": -21.04575538635254,
+      "loss": 0.004329894855618477,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -15.817092895507812,
+      "rewards/margins": 26.2744140625,
+      "rewards/rejected": -42.09151077270508,
+      "step": 340
+    },
+    {
+      "epoch": 0.9641873278236914,
+      "grad_norm": 0.00016785685147624463,
+      "learning_rate": 2.2364217252396165e-06,
+      "logits/chosen": 2.988264322280884,
+      "logits/rejected": 3.05815052986145,
+      "logps/chosen": -8.240289688110352,
+      "logps/rejected": -21.153888702392578,
+      "loss": 0.008007270097732545,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -16.480579376220703,
+      "rewards/margins": 25.827194213867188,
+      "rewards/rejected": -42.307777404785156,
+      "step": 350
+    },
+    {
+      "epoch": 0.9917355371900827,
+      "grad_norm": 3.048083543777466,
+      "learning_rate": 6.389776357827476e-07,
+      "logits/chosen": 3.002927780151367,
+      "logits/rejected": 3.080221652984619,
+      "logps/chosen": -8.972009658813477,
+      "logps/rejected": -20.78287124633789,
+      "loss": 0.08935376405715942,
+      "nll_loss": 0.0,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -17.944019317626953,
+      "rewards/margins": 23.621723175048828,
+      "rewards/rejected": -41.56574249267578,
+      "step": 360
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 363,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-363/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3bae6079ee85f272ee5c1148388807254b89f462d2cec3c58b92c2a1cb918d
+size 5585

eval_results.json

@@ -0,0 +1,14 @@
+{
+  "model_dir": "/workspace/ckpts/simpo-xsum-low-mid-high-0220_0518",
+  "n_pairs": 197,
+  "auroc": 0.9057177458836868,
+  "wins": 193,
+  "ties": 0,
+  "losses": 4,
+  "mean_orig": -8.342779590935699,
+  "mean_misc": -18.936781556189185,
+  "n_style": 0,
+  "brio_mean": NaN,
+  "llama_mean": NaN,
+  "style_gap": NaN
+}

style_control_results_xsum.json

@@ -0,0 +1,10 @@
+{
+  "model_dir": "/workspace/ckpts/simpo-xsum-low-mid-high-0220_0518",
+  "dataset": "xsum",
+  "n": 100,
+  "mean_orig": -8.612068988650032,
+  "mean_gpt_faithful": -23.530381366528278,
+  "mean_misc": -19.799843373021705,
+  "auroc_orig_vs_misc": 0.9271,
+  "auroc_gpt_vs_misc": 0.2939
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "TokenizersBackend"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3bae6079ee85f272ee5c1148388807254b89f462d2cec3c58b92c2a1cb918d
+size 5585