starlineventures/pilot-talk

README.md

@@ -1,10 +1,11 @@
 ---
-base_model: microsoft/phi-2
+base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
 library_name: peft
 license: mit
 tags:
 - trl
 - sft
+- finetuned
 - generated_from_trainer
 model-index:
 - name: outputs
@@ -16,7 +17,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # outputs
 
-This model is a fine-tuned version of [microsoft/phi-2](https://huggingface.co/microsoft/phi-2) on an unknown dataset.
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on an unknown dataset.
 
 ## Model description
 
@@ -36,7 +37,7 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 10
+- train_batch_size: 4
 - eval_batch_size: 16
 - seed: 3407
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08

adapter_config.json

@@ -1,10 +1,10 @@
 {
   "alpha_pattern": {},
   "auto_mapping": {
-    "base_model_class": "PhiForCausalLM",
-    "parent_library": "transformers.models.phi.modeling_phi"
+    "base_model_class": "Qwen2ForCausalLM",
+    "parent_library": "transformers.models.qwen2.modeling_qwen2"
   },
-  "base_model_name_or_path": "microsoft/phi-2",
+  "base_model_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -26,9 +26,9 @@
     "down_proj",
     "k_proj",
     "up_proj",
-    "v_proj",
     "q_proj",
     "gate_proj",
+    "v_proj",
     "o_proj"
   ],
   "task_type": null,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f3a1cea7d969cef21da106d3e9c7d41b5283af202e93bf96911eebc61146215
-size 94422368
+oid sha256:60d95b10b6e140a9626a7058d5038528f2ff80148dc4569b881db56052046509
+size 40

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.125,
+    "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.2409752929911894,
-    "train_runtime": 174.6588,
-    "train_samples_per_second": 13.741,
-    "train_steps_per_second": 1.374
+    "train_loss": 0.11339402707914512,
+    "train_runtime": 443.7998,
+    "train_samples_per_second": 4.326,
+    "train_steps_per_second": 1.082
 }

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 10000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151646,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.44.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0da30790a2243077da2841088a57e8b854924ab9dd99a138cac9fb09043fb8
+size 3554214752

special_tokens_map.json

@@ -1,24 +1,17 @@
 {
   "bos_token": {
-    "content": "<|endoftext|>",
+    "content": "<｜begin▁of▁sentence｜>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<｜end▁of▁sentence｜>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<|endoftext|>",
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
+  "pad_token": "<｜end▁of▁sentence｜>"
 }

tokenizer_config.json

@@ -1,327 +1,194 @@
 {
-  "add_bos_token": false,
-  "add_prefix_space": false,
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
-    "50256": {
-      "content": "<|endoftext|>",
+    "151643": {
+      "content": "<｜end▁of▁sentence｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "50257": {
-      "content": "                               ",
+    "151644": {
+      "content": "<｜User｜>",
       "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50258": {
-      "content": "                              ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50259": {
-      "content": "                             ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50260": {
-      "content": "                            ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50261": {
-      "content": "                           ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50262": {
-      "content": "                          ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50263": {
-      "content": "                         ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50264": {
-      "content": "                        ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50265": {
-      "content": "                       ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50266": {
-      "content": "                      ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50267": {
-      "content": "                     ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50268": {
-      "content": "                    ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50269": {
-      "content": "                   ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50270": {
-      "content": "                  ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50271": {
-      "content": "                 ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50272": {
-      "content": "                ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50273": {
-      "content": "               ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "50274": {
-      "content": "              ",
-      "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50275": {
-      "content": "             ",
+    "151645": {
+      "content": "<｜Assistant｜>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50276": {
-      "content": "            ",
+    "151646": {
+      "content": "<｜begin▁of▁sentence｜>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50277": {
-      "content": "           ",
+    "151647": {
+      "content": "<|EOT|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50278": {
-      "content": "          ",
+    "151648": {
+      "content": "<think>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50279": {
-      "content": "         ",
+    "151649": {
+      "content": "</think>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50280": {
-      "content": "        ",
+    "151650": {
+      "content": "<|quad_start|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50281": {
-      "content": "       ",
+    "151651": {
+      "content": "<|quad_end|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50282": {
-      "content": "      ",
+    "151652": {
+      "content": "<|vision_start|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50283": {
-      "content": "     ",
+    "151653": {
+      "content": "<|vision_end|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50284": {
-      "content": "    ",
+    "151654": {
+      "content": "<|vision_pad|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50285": {
-      "content": "   ",
+    "151655": {
+      "content": "<|image_pad|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50286": {
-      "content": "  ",
+    "151656": {
+      "content": "<|video_pad|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "50287": {
-      "content": "\t\t\t\t\t\t\t\t\t",
+    "151657": {
+      "content": "<tool_call>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50288": {
-      "content": "\t\t\t\t\t\t\t\t",
+    "151658": {
+      "content": "</tool_call>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50289": {
-      "content": "\t\t\t\t\t\t\t",
+    "151659": {
+      "content": "<|fim_prefix|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50290": {
-      "content": "\t\t\t\t\t\t",
+    "151660": {
+      "content": "<|fim_middle|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50291": {
-      "content": "\t\t\t\t\t",
+    "151661": {
+      "content": "<|fim_suffix|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50292": {
-      "content": "\t\t\t\t",
+    "151662": {
+      "content": "<|fim_pad|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50293": {
-      "content": "\t\t\t",
+    "151663": {
+      "content": "<|repo_name|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     },
-    "50294": {
-      "content": "\t\t",
+    "151664": {
+      "content": "<|file_sep|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": false
     }
   },
-  "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
-  "errors": "replace",
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "legacy": true,
   "model_max_length": 2048,
-  "pad_token": "<|endoftext|>",
-  "return_token_type_ids": false,
-  "tokenizer_class": "CodeGenTokenizer",
-  "unk_token": "<|endoftext|>"
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
 }

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.125,
+    "epoch": 3.0,
     "total_flos": 0.0,
-    "train_loss": 0.2409752929911894,
-    "train_runtime": 174.6588,
-    "train_samples_per_second": 13.741,
-    "train_steps_per_second": 1.374
+    "train_loss": 0.11339402707914512,
+    "train_runtime": 443.7998,
+    "train_samples_per_second": 4.326,
+    "train_steps_per_second": 1.082
 }

trainer_state.json

@@ -1,144 +1,389 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.125,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 170,
+  "global_step": 480,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
+    {
+      "epoch": 0.0625,
+      "grad_norm": 0.6764543056488037,
+      "learning_rate": 9.791666666666667e-05,
+      "loss": 4.0135,
+      "step": 10
+    },
     {
       "epoch": 0.125,
-      "grad_norm": 35.11359405517578,
+      "grad_norm": 0.2798255681991577,
       "learning_rate": 9.583333333333334e-05,
-      "loss": 1.3709,
-      "step": 10
+      "loss": 0.1079,
+      "step": 20
+    },
+    {
+      "epoch": 0.1875,
+      "grad_norm": 0.2010556012392044,
+      "learning_rate": 9.375e-05,
+      "loss": 0.0619,
+      "step": 30
     },
     {
       "epoch": 0.25,
-      "grad_norm": 28.059900283813477,
+      "grad_norm": 0.19590723514556885,
       "learning_rate": 9.166666666666667e-05,
-      "loss": 0.4762,
-      "step": 20
+      "loss": 0.0474,
+      "step": 40
+    },
+    {
+      "epoch": 0.3125,
+      "grad_norm": 0.1653718799352646,
+      "learning_rate": 8.958333333333335e-05,
+      "loss": 0.0417,
+      "step": 50
     },
     {
       "epoch": 0.375,
-      "grad_norm": 30.004230499267578,
+      "grad_norm": 0.19019761681556702,
       "learning_rate": 8.75e-05,
-      "loss": 0.2966,
-      "step": 30
+      "loss": 0.037,
+      "step": 60
+    },
+    {
+      "epoch": 0.4375,
+      "grad_norm": 0.18890416622161865,
+      "learning_rate": 8.541666666666666e-05,
+      "loss": 0.0339,
+      "step": 70
     },
     {
       "epoch": 0.5,
-      "grad_norm": 30.531251907348633,
+      "grad_norm": 0.19193948805332184,
       "learning_rate": 8.333333333333334e-05,
-      "loss": 0.2166,
-      "step": 40
+      "loss": 0.0317,
+      "step": 80
+    },
+    {
+      "epoch": 0.5625,
+      "grad_norm": 0.12777990102767944,
+      "learning_rate": 8.125000000000001e-05,
+      "loss": 0.0301,
+      "step": 90
     },
     {
       "epoch": 0.625,
-      "grad_norm": 25.447418212890625,
+      "grad_norm": 0.12041394412517548,
       "learning_rate": 7.916666666666666e-05,
-      "loss": 0.1816,
-      "step": 50
+      "loss": 0.0287,
+      "step": 100
+    },
+    {
+      "epoch": 0.6875,
+      "grad_norm": 0.1440989226102829,
+      "learning_rate": 7.708333333333334e-05,
+      "loss": 0.0288,
+      "step": 110
+    },
+    {
+      "epoch": 0.6875,
+      "eval_runtime": 12.6929,
+      "eval_samples_per_second": 12.606,
+      "eval_steps_per_second": 0.788,
+      "step": 110
     },
     {
       "epoch": 0.75,
-      "grad_norm": 25.023717880249023,
+      "grad_norm": 0.14094920456409454,
       "learning_rate": 7.500000000000001e-05,
-      "loss": 0.1653,
-      "step": 60
+      "loss": 0.0284,
+      "step": 120
+    },
+    {
+      "epoch": 0.8125,
+      "grad_norm": 0.13362859189510345,
+      "learning_rate": 7.291666666666667e-05,
+      "loss": 0.028,
+      "step": 130
     },
     {
       "epoch": 0.875,
-      "grad_norm": 23.13313102722168,
+      "grad_norm": 0.10230981558561325,
       "learning_rate": 7.083333333333334e-05,
-      "loss": 0.1506,
-      "step": 70
+      "loss": 0.0278,
+      "step": 140
+    },
+    {
+      "epoch": 0.9375,
+      "grad_norm": 0.10484622418880463,
+      "learning_rate": 6.875e-05,
+      "loss": 0.0277,
+      "step": 150
     },
     {
       "epoch": 1.0,
-      "grad_norm": 21.35039710998535,
+      "grad_norm": 0.1045694574713707,
       "learning_rate": 6.666666666666667e-05,
-      "loss": 0.1369,
-      "step": 80
+      "loss": 0.0277,
+      "step": 160
+    },
+    {
+      "epoch": 1.0,
+      "eval_runtime": 11.2891,
+      "eval_samples_per_second": 14.173,
+      "eval_steps_per_second": 0.886,
+      "step": 160
+    },
+    {
+      "epoch": 1.0625,
+      "grad_norm": 0.11217521131038666,
+      "learning_rate": 6.458333333333334e-05,
+      "loss": 0.0267,
+      "step": 170
     },
     {
       "epoch": 1.125,
-      "grad_norm": 19.913026809692383,
+      "grad_norm": 0.1110721006989479,
       "learning_rate": 6.25e-05,
-      "loss": 0.1315,
-      "step": 90
+      "loss": 0.027,
+      "step": 180
+    },
+    {
+      "epoch": 1.1875,
+      "grad_norm": 0.12359625101089478,
+      "learning_rate": 6.041666666666667e-05,
+      "loss": 0.0272,
+      "step": 190
     },
     {
       "epoch": 1.25,
-      "grad_norm": 21.789073944091797,
+      "grad_norm": 0.10066195577383041,
       "learning_rate": 5.833333333333334e-05,
-      "loss": 0.1296,
-      "step": 100
+      "loss": 0.027,
+      "step": 200
+    },
+    {
+      "epoch": 1.3125,
+      "grad_norm": 0.10776817798614502,
+      "learning_rate": 5.6250000000000005e-05,
+      "loss": 0.0269,
+      "step": 210
     },
     {
       "epoch": 1.375,
-      "grad_norm": 25.12034797668457,
+      "grad_norm": 0.10342445224523544,
       "learning_rate": 5.4166666666666664e-05,
-      "loss": 0.1279,
-      "step": 110
+      "loss": 0.027,
+      "step": 220
+    },
+    {
+      "epoch": 1.4375,
+      "grad_norm": 0.0953444242477417,
+      "learning_rate": 5.208333333333334e-05,
+      "loss": 0.027,
+      "step": 230
     },
     {
       "epoch": 1.5,
-      "grad_norm": 25.35553550720215,
+      "grad_norm": 0.10930311679840088,
       "learning_rate": 5e-05,
-      "loss": 0.1241,
-      "step": 120
+      "loss": 0.0269,
+      "step": 240
+    },
+    {
+      "epoch": 1.5625,
+      "grad_norm": 0.11344899982213974,
+      "learning_rate": 4.791666666666667e-05,
+      "loss": 0.0268,
+      "step": 250
     },
     {
       "epoch": 1.625,
-      "grad_norm": 23.74753761291504,
+      "grad_norm": 0.10314708203077316,
       "learning_rate": 4.5833333333333334e-05,
-      "loss": 0.1208,
-      "step": 130
+      "loss": 0.0267,
+      "step": 260
+    },
+    {
+      "epoch": 1.6875,
+      "grad_norm": 0.11634312570095062,
+      "learning_rate": 4.375e-05,
+      "loss": 0.0266,
+      "step": 270
     },
     {
       "epoch": 1.75,
-      "grad_norm": 23.058244705200195,
+      "grad_norm": 0.09650956094264984,
       "learning_rate": 4.166666666666667e-05,
-      "loss": 0.1188,
-      "step": 140
+      "loss": 0.0267,
+      "step": 280
+    },
+    {
+      "epoch": 1.8125,
+      "grad_norm": 0.10790986567735672,
+      "learning_rate": 3.958333333333333e-05,
+      "loss": 0.0266,
+      "step": 290
     },
     {
       "epoch": 1.875,
-      "grad_norm": 22.608642578125,
+      "grad_norm": 0.10706381499767303,
       "learning_rate": 3.7500000000000003e-05,
-      "loss": 0.1174,
-      "step": 150
+      "loss": 0.0264,
+      "step": 300
+    },
+    {
+      "epoch": 1.9375,
+      "grad_norm": 0.11093062162399292,
+      "learning_rate": 3.541666666666667e-05,
+      "loss": 0.0266,
+      "step": 310
     },
     {
       "epoch": 2.0,
-      "grad_norm": 22.311994552612305,
+      "grad_norm": 0.10339082777500153,
       "learning_rate": 3.3333333333333335e-05,
-      "loss": 0.1162,
-      "step": 160
+      "loss": 0.0265,
+      "step": 320
+    },
+    {
+      "epoch": 2.0,
+      "eval_runtime": 11.299,
+      "eval_samples_per_second": 14.161,
+      "eval_steps_per_second": 0.885,
+      "step": 320
+    },
+    {
+      "epoch": 2.0625,
+      "grad_norm": 0.10130282491445541,
+      "learning_rate": 3.125e-05,
+      "loss": 0.026,
+      "step": 330
     },
     {
       "epoch": 2.125,
-      "grad_norm": 23.028535842895508,
+      "grad_norm": 0.10614955425262451,
       "learning_rate": 2.916666666666667e-05,
-      "loss": 0.1158,
-      "step": 170
+      "loss": 0.0257,
+      "step": 340
     },
     {
-      "epoch": 2.125,
-      "step": 170,
+      "epoch": 2.1875,
+      "grad_norm": 0.1143997386097908,
+      "learning_rate": 2.7083333333333332e-05,
+      "loss": 0.0261,
+      "step": 350
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 0.10745866596698761,
+      "learning_rate": 2.5e-05,
+      "loss": 0.0256,
+      "step": 360
+    },
+    {
+      "epoch": 2.3125,
+      "grad_norm": 0.1030006930232048,
+      "learning_rate": 2.2916666666666667e-05,
+      "loss": 0.0258,
+      "step": 370
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.10611753165721893,
+      "learning_rate": 2.0833333333333336e-05,
+      "loss": 0.026,
+      "step": 380
+    },
+    {
+      "epoch": 2.4375,
+      "grad_norm": 0.10169661045074463,
+      "learning_rate": 1.8750000000000002e-05,
+      "loss": 0.0257,
+      "step": 390
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.11938793212175369,
+      "learning_rate": 1.6666666666666667e-05,
+      "loss": 0.0258,
+      "step": 400
+    },
+    {
+      "epoch": 2.5625,
+      "grad_norm": 0.0978316143155098,
+      "learning_rate": 1.4583333333333335e-05,
+      "loss": 0.0259,
+      "step": 410
+    },
+    {
+      "epoch": 2.625,
+      "grad_norm": 0.09611309319734573,
+      "learning_rate": 1.25e-05,
+      "loss": 0.0258,
+      "step": 420
+    },
+    {
+      "epoch": 2.6875,
+      "grad_norm": 0.10421048849821091,
+      "learning_rate": 1.0416666666666668e-05,
+      "loss": 0.0258,
+      "step": 430
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 0.09502692520618439,
+      "learning_rate": 8.333333333333334e-06,
+      "loss": 0.0257,
+      "step": 440
+    },
+    {
+      "epoch": 2.8125,
+      "grad_norm": 0.10091052949428558,
+      "learning_rate": 6.25e-06,
+      "loss": 0.0256,
+      "step": 450
+    },
+    {
+      "epoch": 2.875,
+      "grad_norm": 0.10061470419168472,
+      "learning_rate": 4.166666666666667e-06,
+      "loss": 0.0257,
+      "step": 460
+    },
+    {
+      "epoch": 2.9375,
+      "grad_norm": 0.09805355966091156,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0255,
+      "step": 470
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.11753270030021667,
+      "learning_rate": 0.0,
+      "loss": 0.0256,
+      "step": 480
+    },
+    {
+      "epoch": 3.0,
+      "eval_runtime": 11.3126,
+      "eval_samples_per_second": 14.144,
+      "eval_steps_per_second": 0.884,
+      "step": 480
+    },
+    {
+      "epoch": 3.0,
+      "step": 480,
       "total_flos": 0.0,
-      "train_loss": 0.2409752929911894,
-      "train_runtime": 174.6588,
-      "train_samples_per_second": 13.741,
-      "train_steps_per_second": 1.374
+      "train_loss": 0.11339402707914512,
+      "train_runtime": 443.7998,
+      "train_samples_per_second": 4.326,
+      "train_steps_per_second": 1.082
     }
   ],
   "logging_steps": 10,
-  "max_steps": 240,
+  "max_steps": 480,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 3,
   "save_steps": 500,
@@ -155,7 +400,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 10,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0bf77fa574b65faae23151f91dc99b2b3a380810ab2b3e9268e86e0152036049
+oid sha256:d3ea36db01dbb1c5915e1ff553ab1d2d31cd6119900ddbf872951d161132b290
 size 5432