starlineventures/pilot-talk

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,57 +1,56 @@
 ---
 base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
-library_name: peft
-license: mit
+library_name: transformers
+model_name: outputs
 tags:
-- trl
-- sft
-- finetuned
 - generated_from_trainer
-model-index:
-- name: outputs
-  results: []
+licence: license
 ---
 
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
+# Model Card for outputs
 
-# outputs
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B).
+It has been trained using [TRL](https://github.com/huggingface/trl).
 
-This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on an unknown dataset.
+## Quick start
 
-## Model description
+```python
+from transformers import pipeline
 
-More information needed
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="starlineventures/outputs", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
 
-## Intended uses & limitations
-
-More information needed
-
-## Training and evaluation data
+## Training procedure
 
-More information needed
+ 
 
-## Training procedure
 
-### Training hyperparameters
+This model was trained with SFT.
 
-The following hyperparameters were used during training:
-- learning_rate: 0.0001
-- train_batch_size: 4
-- eval_batch_size: 16
-- seed: 3407
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: linear
-- num_epochs: 3
+### Framework versions
 
-### Training results
+- TRL: 0.18.1
+- Transformers: 4.52.4
+- Pytorch: 2.7.0
+- Datasets: 3.6.0
+- Tokenizers: 0.21.1
 
+## Citations
 
 
-### Framework versions
 
-- PEFT 0.12.0
-- Transformers 4.44.2
-- Pytorch 2.4.0+cu121
-- Datasets 3.0.0
-- Tokenizers 0.19.1
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

adapter_config.json

@@ -6,6 +6,9 @@
   },
   "base_model_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
   "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
@@ -14,6 +17,7 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 16,
+  "lora_bias": false,
   "lora_dropout": 0,
   "megatron_config": null,
   "megatron_core": "megatron.core",
@@ -23,15 +27,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "k_proj",
-    "up_proj",
-    "q_proj",
     "gate_proj",
     "v_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "down_proj",
     "o_proj"
   ],
   "task_type": null,
+  "trainable_token_indices": null,
   "use_dora": false,
   "use_rslora": false
 }

all_results.json

@@ -1,8 +1,7 @@
 {
-    "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.11339402707914512,
-    "train_runtime": 443.7998,
-    "train_samples_per_second": 4.326,
-    "train_steps_per_second": 1.082
+    "train_loss": 0.09647794626653194,
+    "train_runtime": 276.4027,
+    "train_samples_per_second": 4.631,
+    "train_steps_per_second": 1.158
 }

chat_template.jinja

@@ -0,0 +1 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\n'}}{% endif %}

config.json

@@ -1,5 +1,4 @@
 {
-  "_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
@@ -17,11 +16,12 @@
   "num_hidden_layers": 28,
   "num_key_value_heads": 2,
   "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
   "rope_theta": 10000,
-  "sliding_window": null,
+  "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.44.2",
+  "transformers_version": "4.52.4",
   "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,

generation_config.json

@@ -5,5 +5,5 @@
   "eos_token_id": 151643,
   "temperature": 0.6,
   "top_p": 0.95,
-  "transformers_version": "4.44.2"
+  "transformers_version": "4.52.4"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef0da30790a2243077da2841088a57e8b854924ab9dd99a138cac9fb09043fb8
+oid sha256:25afe2c83203d89e709118ec4867d6c1d40d0479a8296b4f90f5090578885517
 size 3554214752

tokenizer_config.json

@@ -181,14 +181,14 @@
     }
   },
   "bos_token": "<｜begin▁of▁sentence｜>",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<｜end▁of▁sentence｜>",
+  "extra_special_tokens": {},
   "legacy": true,
   "model_max_length": 2048,
   "pad_token": "<｜end▁of▁sentence｜>",
   "sp_model_kwargs": {},
-  "tokenizer_class": "LlamaTokenizer",
+  "tokenizer_class": "LlamaTokenizerFast",
   "unk_token": null,
   "use_default_system_prompt": false
 }

train_results.json

@@ -1,8 +1,7 @@
 {
-    "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.11339402707914512,
-    "train_runtime": 443.7998,
-    "train_samples_per_second": 4.326,
-    "train_steps_per_second": 1.082
+    "train_loss": 0.09647794626653194,
+    "train_runtime": 276.4027,
+    "train_samples_per_second": 4.631,
+    "train_steps_per_second": 1.158
 }

trainer_state.json

@@ -1,391 +1,337 @@
 {
+  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 480,
+  "global_step": 320,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.0625,
-      "grad_norm": 0.6764543056488037,
-      "learning_rate": 9.791666666666667e-05,
-      "loss": 4.0135,
+      "grad_norm": 0.6001169085502625,
+      "learning_rate": 9.718750000000001e-05,
+      "loss": 1.9881,
+      "mean_token_accuracy": 0.7977039694786072,
+      "num_tokens": 81920.0,
       "step": 10
     },
     {
       "epoch": 0.125,
-      "grad_norm": 0.2798255681991577,
-      "learning_rate": 9.583333333333334e-05,
-      "loss": 0.1079,
+      "grad_norm": 0.46209943294525146,
+      "learning_rate": 9.40625e-05,
+      "loss": 0.1785,
+      "mean_token_accuracy": 0.9618344008922577,
+      "num_tokens": 163840.0,
       "step": 20
     },
     {
       "epoch": 0.1875,
-      "grad_norm": 0.2010556012392044,
-      "learning_rate": 9.375e-05,
-      "loss": 0.0619,
+      "grad_norm": 0.25115829706192017,
+      "learning_rate": 9.093750000000001e-05,
+      "loss": 0.075,
+      "mean_token_accuracy": 0.9808378159999848,
+      "num_tokens": 245760.0,
       "step": 30
     },
     {
       "epoch": 0.25,
-      "grad_norm": 0.19590723514556885,
-      "learning_rate": 9.166666666666667e-05,
-      "loss": 0.0474,
+      "grad_norm": 0.18340203166007996,
+      "learning_rate": 8.781250000000001e-05,
+      "loss": 0.0518,
+      "mean_token_accuracy": 0.98480703830719,
+      "num_tokens": 327680.0,
       "step": 40
     },
     {
       "epoch": 0.3125,
-      "grad_norm": 0.1653718799352646,
-      "learning_rate": 8.958333333333335e-05,
-      "loss": 0.0417,
+      "grad_norm": 0.16958290338516235,
+      "learning_rate": 8.46875e-05,
+      "loss": 0.0439,
+      "mean_token_accuracy": 0.9856253087520599,
+      "num_tokens": 409600.0,
       "step": 50
     },
     {
       "epoch": 0.375,
-      "grad_norm": 0.19019761681556702,
-      "learning_rate": 8.75e-05,
-      "loss": 0.037,
+      "grad_norm": 0.18090437352657318,
+      "learning_rate": 8.156250000000001e-05,
+      "loss": 0.0362,
+      "mean_token_accuracy": 0.9867611169815064,
+      "num_tokens": 491520.0,
       "step": 60
     },
     {
       "epoch": 0.4375,
-      "grad_norm": 0.18890416622161865,
-      "learning_rate": 8.541666666666666e-05,
-      "loss": 0.0339,
+      "grad_norm": 0.14541096985340118,
+      "learning_rate": 7.84375e-05,
+      "loss": 0.0324,
+      "mean_token_accuracy": 0.9876038134098053,
+      "num_tokens": 573440.0,
       "step": 70
     },
     {
       "epoch": 0.5,
-      "grad_norm": 0.19193948805332184,
-      "learning_rate": 8.333333333333334e-05,
-      "loss": 0.0317,
+      "grad_norm": 0.3248767852783203,
+      "learning_rate": 7.531250000000001e-05,
+      "loss": 0.0309,
+      "mean_token_accuracy": 0.9874206185340881,
+      "num_tokens": 655360.0,
       "step": 80
     },
     {
       "epoch": 0.5625,
-      "grad_norm": 0.12777990102767944,
-      "learning_rate": 8.125000000000001e-05,
-      "loss": 0.0301,
+      "grad_norm": 0.12345177680253983,
+      "learning_rate": 7.218750000000001e-05,
+      "loss": 0.03,
+      "mean_token_accuracy": 0.9874084055423736,
+      "num_tokens": 737280.0,
       "step": 90
     },
     {
       "epoch": 0.625,
-      "grad_norm": 0.12041394412517548,
-      "learning_rate": 7.916666666666666e-05,
-      "loss": 0.0287,
+      "grad_norm": 0.12568464875221252,
+      "learning_rate": 6.90625e-05,
+      "loss": 0.0288,
+      "mean_token_accuracy": 0.9879335641860962,
+      "num_tokens": 819200.0,
       "step": 100
     },
     {
       "epoch": 0.6875,
-      "grad_norm": 0.1440989226102829,
-      "learning_rate": 7.708333333333334e-05,
+      "grad_norm": 0.11818379908800125,
+      "learning_rate": 6.593750000000001e-05,
       "loss": 0.0288,
+      "mean_token_accuracy": 0.9873839795589447,
+      "num_tokens": 901120.0,
       "step": 110
     },
     {
       "epoch": 0.6875,
-      "eval_runtime": 12.6929,
-      "eval_samples_per_second": 12.606,
-      "eval_steps_per_second": 0.788,
+      "eval_runtime": 11.8942,
+      "eval_samples_per_second": 13.452,
+      "eval_steps_per_second": 0.841,
       "step": 110
     },
     {
       "epoch": 0.75,
-      "grad_norm": 0.14094920456409454,
-      "learning_rate": 7.500000000000001e-05,
-      "loss": 0.0284,
+      "grad_norm": 0.12166234850883484,
+      "learning_rate": 6.28125e-05,
+      "loss": 0.0282,
+      "mean_token_accuracy": 0.9881167590618134,
+      "num_tokens": 983040.0,
       "step": 120
     },
     {
       "epoch": 0.8125,
-      "grad_norm": 0.13362859189510345,
-      "learning_rate": 7.291666666666667e-05,
-      "loss": 0.028,
+      "grad_norm": 0.1256779283285141,
+      "learning_rate": 5.968750000000001e-05,
+      "loss": 0.0279,
+      "mean_token_accuracy": 0.9878847122192382,
+      "num_tokens": 1064960.0,
       "step": 130
     },
     {
       "epoch": 0.875,
-      "grad_norm": 0.10230981558561325,
-      "learning_rate": 7.083333333333334e-05,
-      "loss": 0.0278,
+      "grad_norm": 0.11304116994142532,
+      "learning_rate": 5.6562500000000006e-05,
+      "loss": 0.0276,
+      "mean_token_accuracy": 0.987799221277237,
+      "num_tokens": 1146880.0,
       "step": 140
     },
     {
       "epoch": 0.9375,
-      "grad_norm": 0.10484622418880463,
-      "learning_rate": 6.875e-05,
-      "loss": 0.0277,
+      "grad_norm": 0.10594538599252701,
+      "learning_rate": 5.3437500000000005e-05,
+      "loss": 0.0276,
+      "mean_token_accuracy": 0.9880923330783844,
+      "num_tokens": 1228800.0,
       "step": 150
     },
     {
       "epoch": 1.0,
-      "grad_norm": 0.1045694574713707,
-      "learning_rate": 6.666666666666667e-05,
-      "loss": 0.0277,
+      "grad_norm": 0.1052316427230835,
+      "learning_rate": 5.031250000000001e-05,
+      "loss": 0.0274,
+      "mean_token_accuracy": 0.9881289720535278,
+      "num_tokens": 1310720.0,
       "step": 160
     },
     {
       "epoch": 1.0,
-      "eval_runtime": 11.2891,
-      "eval_samples_per_second": 14.173,
-      "eval_steps_per_second": 0.886,
+      "eval_runtime": 10.8105,
+      "eval_samples_per_second": 14.8,
+      "eval_steps_per_second": 0.925,
       "step": 160
     },
     {
       "epoch": 1.0625,
-      "grad_norm": 0.11217521131038666,
-      "learning_rate": 6.458333333333334e-05,
+      "grad_norm": 0.11138579249382019,
+      "learning_rate": 4.71875e-05,
       "loss": 0.0267,
+      "mean_token_accuracy": 0.9883243799209595,
+      "num_tokens": 1392640.0,
       "step": 170
     },
     {
       "epoch": 1.125,
-      "grad_norm": 0.1110721006989479,
-      "learning_rate": 6.25e-05,
+      "grad_norm": 0.11131029576063156,
+      "learning_rate": 4.40625e-05,
       "loss": 0.027,
+      "mean_token_accuracy": 0.9881411850452423,
+      "num_tokens": 1474560.0,
       "step": 180
     },
     {
       "epoch": 1.1875,
-      "grad_norm": 0.12359625101089478,
-      "learning_rate": 6.041666666666667e-05,
-      "loss": 0.0272,
+      "grad_norm": 0.11658758670091629,
+      "learning_rate": 4.09375e-05,
+      "loss": 0.0269,
+      "mean_token_accuracy": 0.9882755279541016,
+      "num_tokens": 1556480.0,
       "step": 190
     },
     {
       "epoch": 1.25,
-      "grad_norm": 0.10066195577383041,
-      "learning_rate": 5.833333333333334e-05,
-      "loss": 0.027,
+      "grad_norm": 0.11032383143901825,
+      "learning_rate": 3.78125e-05,
+      "loss": 0.0266,
+      "mean_token_accuracy": 0.9884098708629608,
+      "num_tokens": 1638400.0,
       "step": 200
     },
     {
       "epoch": 1.3125,
-      "grad_norm": 0.10776817798614502,
-      "learning_rate": 5.6250000000000005e-05,
-      "loss": 0.0269,
+      "grad_norm": 0.10452437400817871,
+      "learning_rate": 3.46875e-05,
+      "loss": 0.0267,
+      "mean_token_accuracy": 0.9882388889789582,
+      "num_tokens": 1720320.0,
       "step": 210
     },
     {
       "epoch": 1.375,
-      "grad_norm": 0.10342445224523544,
-      "learning_rate": 5.4166666666666664e-05,
-      "loss": 0.027,
+      "grad_norm": 0.10454697161912918,
+      "learning_rate": 3.15625e-05,
+      "loss": 0.0267,
+      "mean_token_accuracy": 0.9878969252109527,
+      "num_tokens": 1802240.0,
       "step": 220
     },
     {
       "epoch": 1.4375,
-      "grad_norm": 0.0953444242477417,
-      "learning_rate": 5.208333333333334e-05,
-      "loss": 0.027,
+      "grad_norm": 0.09663768112659454,
+      "learning_rate": 2.84375e-05,
+      "loss": 0.0266,
+      "mean_token_accuracy": 0.9883976578712463,
+      "num_tokens": 1884160.0,
       "step": 230
     },
     {
       "epoch": 1.5,
-      "grad_norm": 0.10930311679840088,
-      "learning_rate": 5e-05,
-      "loss": 0.0269,
+      "grad_norm": 0.11553214490413666,
+      "learning_rate": 2.53125e-05,
+      "loss": 0.0267,
+      "mean_token_accuracy": 0.9884098708629608,
+      "num_tokens": 1966080.0,
       "step": 240
     },
     {
       "epoch": 1.5625,
-      "grad_norm": 0.11344899982213974,
-      "learning_rate": 4.791666666666667e-05,
-      "loss": 0.0268,
+      "grad_norm": 0.11412502825260162,
+      "learning_rate": 2.21875e-05,
+      "loss": 0.0263,
+      "mean_token_accuracy": 0.9883854448795318,
+      "num_tokens": 2048000.0,
       "step": 250
     },
     {
       "epoch": 1.625,
-      "grad_norm": 0.10314708203077316,
-      "learning_rate": 4.5833333333333334e-05,
-      "loss": 0.0267,
+      "grad_norm": 0.1027199923992157,
+      "learning_rate": 1.90625e-05,
+      "loss": 0.0264,
+      "mean_token_accuracy": 0.9882999539375306,
+      "num_tokens": 2129920.0,
       "step": 260
     },
     {
       "epoch": 1.6875,
-      "grad_norm": 0.11634312570095062,
-      "learning_rate": 4.375e-05,
-      "loss": 0.0266,
+      "grad_norm": 0.12076633423566818,
+      "learning_rate": 1.59375e-05,
+      "loss": 0.0264,
+      "mean_token_accuracy": 0.9882266759872437,
+      "num_tokens": 2211840.0,
       "step": 270
     },
     {
       "epoch": 1.75,
-      "grad_norm": 0.09650956094264984,
-      "learning_rate": 4.166666666666667e-05,
-      "loss": 0.0267,
+      "grad_norm": 0.1111675500869751,
+      "learning_rate": 1.28125e-05,
+      "loss": 0.0264,
+      "mean_token_accuracy": 0.9879335641860962,
+      "num_tokens": 2293760.0,
       "step": 280
     },
     {
       "epoch": 1.8125,
-      "grad_norm": 0.10790986567735672,
-      "learning_rate": 3.958333333333333e-05,
-      "loss": 0.0266,
+      "grad_norm": 0.1163574606180191,
+      "learning_rate": 9.6875e-06,
+      "loss": 0.0262,
+      "mean_token_accuracy": 0.9885197877883911,
+      "num_tokens": 2375680.0,
       "step": 290
     },
     {
       "epoch": 1.875,
-      "grad_norm": 0.10706381499767303,
-      "learning_rate": 3.7500000000000003e-05,
-      "loss": 0.0264,
+      "grad_norm": 0.11461054533720016,
+      "learning_rate": 6.5625e-06,
+      "loss": 0.0261,
+      "mean_token_accuracy": 0.988593065738678,
+      "num_tokens": 2457600.0,
       "step": 300
     },
     {
       "epoch": 1.9375,
-      "grad_norm": 0.11093062162399292,
-      "learning_rate": 3.541666666666667e-05,
-      "loss": 0.0266,
+      "grad_norm": 0.10500755161046982,
+      "learning_rate": 3.4375000000000005e-06,
+      "loss": 0.0263,
+      "mean_token_accuracy": 0.9882633149623871,
+      "num_tokens": 2539520.0,
       "step": 310
     },
     {
       "epoch": 2.0,
-      "grad_norm": 0.10339082777500153,
-      "learning_rate": 3.3333333333333335e-05,
-      "loss": 0.0265,
+      "grad_norm": 0.11479007452726364,
+      "learning_rate": 3.125e-07,
+      "loss": 0.0262,
+      "mean_token_accuracy": 0.988336592912674,
+      "num_tokens": 2621440.0,
       "step": 320
     },
     {
       "epoch": 2.0,
-      "eval_runtime": 11.299,
-      "eval_samples_per_second": 14.161,
-      "eval_steps_per_second": 0.885,
+      "eval_runtime": 10.8162,
+      "eval_samples_per_second": 14.793,
+      "eval_steps_per_second": 0.925,
       "step": 320
     },
     {
-      "epoch": 2.0625,
-      "grad_norm": 0.10130282491445541,
-      "learning_rate": 3.125e-05,
-      "loss": 0.026,
-      "step": 330
-    },
-    {
-      "epoch": 2.125,
-      "grad_norm": 0.10614955425262451,
-      "learning_rate": 2.916666666666667e-05,
-      "loss": 0.0257,
-      "step": 340
-    },
-    {
-      "epoch": 2.1875,
-      "grad_norm": 0.1143997386097908,
-      "learning_rate": 2.7083333333333332e-05,
-      "loss": 0.0261,
-      "step": 350
-    },
-    {
-      "epoch": 2.25,
-      "grad_norm": 0.10745866596698761,
-      "learning_rate": 2.5e-05,
-      "loss": 0.0256,
-      "step": 360
-    },
-    {
-      "epoch": 2.3125,
-      "grad_norm": 0.1030006930232048,
-      "learning_rate": 2.2916666666666667e-05,
-      "loss": 0.0258,
-      "step": 370
-    },
-    {
-      "epoch": 2.375,
-      "grad_norm": 0.10611753165721893,
-      "learning_rate": 2.0833333333333336e-05,
-      "loss": 0.026,
-      "step": 380
-    },
-    {
-      "epoch": 2.4375,
-      "grad_norm": 0.10169661045074463,
-      "learning_rate": 1.8750000000000002e-05,
-      "loss": 0.0257,
-      "step": 390
-    },
-    {
-      "epoch": 2.5,
-      "grad_norm": 0.11938793212175369,
-      "learning_rate": 1.6666666666666667e-05,
-      "loss": 0.0258,
-      "step": 400
-    },
-    {
-      "epoch": 2.5625,
-      "grad_norm": 0.0978316143155098,
-      "learning_rate": 1.4583333333333335e-05,
-      "loss": 0.0259,
-      "step": 410
-    },
-    {
-      "epoch": 2.625,
-      "grad_norm": 0.09611309319734573,
-      "learning_rate": 1.25e-05,
-      "loss": 0.0258,
-      "step": 420
-    },
-    {
-      "epoch": 2.6875,
-      "grad_norm": 0.10421048849821091,
-      "learning_rate": 1.0416666666666668e-05,
-      "loss": 0.0258,
-      "step": 430
-    },
-    {
-      "epoch": 2.75,
-      "grad_norm": 0.09502692520618439,
-      "learning_rate": 8.333333333333334e-06,
-      "loss": 0.0257,
-      "step": 440
-    },
-    {
-      "epoch": 2.8125,
-      "grad_norm": 0.10091052949428558,
-      "learning_rate": 6.25e-06,
-      "loss": 0.0256,
-      "step": 450
-    },
-    {
-      "epoch": 2.875,
-      "grad_norm": 0.10061470419168472,
-      "learning_rate": 4.166666666666667e-06,
-      "loss": 0.0257,
-      "step": 460
-    },
-    {
-      "epoch": 2.9375,
-      "grad_norm": 0.09805355966091156,
-      "learning_rate": 2.0833333333333334e-06,
-      "loss": 0.0255,
-      "step": 470
-    },
-    {
-      "epoch": 3.0,
-      "grad_norm": 0.11753270030021667,
-      "learning_rate": 0.0,
-      "loss": 0.0256,
-      "step": 480
-    },
-    {
-      "epoch": 3.0,
-      "eval_runtime": 11.3126,
-      "eval_samples_per_second": 14.144,
-      "eval_steps_per_second": 0.884,
-      "step": 480
-    },
-    {
-      "epoch": 3.0,
-      "step": 480,
+      "epoch": 2.0,
+      "step": 320,
       "total_flos": 0.0,
-      "train_loss": 0.11339402707914512,
-      "train_runtime": 443.7998,
-      "train_samples_per_second": 4.326,
-      "train_steps_per_second": 1.082
+      "train_loss": 0.09647794626653194,
+      "train_runtime": 276.4027,
+      "train_samples_per_second": 4.631,
+      "train_steps_per_second": 1.158
     }
   ],
   "logging_steps": 10,
-  "max_steps": 480,
+  "max_steps": 320,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d3ea36db01dbb1c5915e1ff553ab1d2d31cd6119900ddbf872951d161132b290
-size 5432
+oid sha256:a2d1ad38d9466f8a2da26fdcc2bc0cfe7f4da2baa9299dcd78d391608ab03e27
+size 6033