Model save

README.md

@@ -0,0 +1,65 @@
+---
+base_model: microsoft/phi-1_5
+datasets:
+- samsum
+library_name: peft
+license: mit
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: test_model
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# test_model
+
+This model is a fine-tuned version of [microsoft/phi-1_5](https://huggingface.co/microsoft/phi-1_5) on the samsum dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.0429
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 2.8377        | 1.0   | 250  | 3.0473          |
+| 2.9821        | 2.0   | 500  | 3.0425          |
+| 2.9724        | 3.0   | 750  | 3.0429          |
+
+
+### Framework versions
+
+- PEFT 0.11.1
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "microsoft/phi-1_5",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "q_proj",
+    "o_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81d92e3783982e3886c76e360c7a4363660259262a76f4eab77e5e9446170d2a
+size 9456424

added_tokens.json

@@ -0,0 +1,40 @@
+{
+  "\t\t": 50294,
+  "\t\t\t": 50293,
+  "\t\t\t\t": 50292,
+  "\t\t\t\t\t": 50291,
+  "\t\t\t\t\t\t": 50290,
+  "\t\t\t\t\t\t\t": 50289,
+  "\t\t\t\t\t\t\t\t": 50288,
+  "\t\t\t\t\t\t\t\t\t": 50287,
+  "  ": 50286,
+  "   ": 50285,
+  "    ": 50284,
+  "     ": 50283,
+  "      ": 50282,
+  "       ": 50281,
+  "        ": 50280,
+  "         ": 50279,
+  "          ": 50278,
+  "           ": 50277,
+  "            ": 50276,
+  "             ": 50275,
+  "              ": 50274,
+  "               ": 50273,
+  "                ": 50272,
+  "                 ": 50271,
+  "                  ": 50270,
+  "                   ": 50269,
+  "                    ": 50268,
+  "                     ": 50267,
+  "                      ": 50266,
+  "                       ": 50265,
+  "                        ": 50264,
+  "                         ": 50263,
+  "                          ": 50262,
+  "                           ": 50261,
+  "                            ": 50260,
+  "                             ": 50259,
+  "                              ": 50258,
+  "                               ": 50257
+}

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 3.042874813079834,
+    "eval_runtime": 37.1738,
+    "eval_samples": 100,
+    "eval_samples_per_second": 2.69,
+    "eval_steps_per_second": 1.345,
+    "total_flos": 3617591393476608.0,
+    "train_loss": 3.0439649012883505,
+    "train_runtime": 1336.343,
+    "train_samples": 500,
+    "train_samples_per_second": 1.122,
+    "train_steps_per_second": 0.561
+}

eval_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "eval_loss": 3.042874813079834,
+    "eval_runtime": 37.1738,
+    "eval_samples": 100,
+    "eval_samples_per_second": 2.69,
+    "eval_steps_per_second": 1.345
+}

runs/Jul12_13-34-37_7d0228143436/events.out.tfevents.1720791287.7d0228143436.1500.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59aa4605d387403b4021e2f4d597aab425e61d1b2793de5c1864e07aa0c590a8
+size 5554

runs/Jul12_13-49-41_7d0228143436/events.out.tfevents.1720792190.7d0228143436.1500.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51e16754df33165734d85632ae82cc83ca005cf86435aeaabe758ca1e3058e66
+size 38204

runs/Jul12_13-49-41_7d0228143436/events.out.tfevents.1720793564.7d0228143436.1500.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6650f48b7255fd3215ffe6cd28ec8d752151a0198efbae9c46945407d9d64ccb
+size 359

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,326 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "                               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "                         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50280": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50281": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50282": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50283": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50284": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50285": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### Input: \n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '### Instruction: \n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Response: \n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Response:' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "return_token_type_ids": false,
+  "tokenizer_class": "CodeGenTokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 3617591393476608.0,
+    "train_loss": 3.0439649012883505,
+    "train_runtime": 1336.343,
+    "train_samples": 500,
+    "train_samples_per_second": 1.122,
+    "train_steps_per_second": 0.561
+}

trainer_state.json

@@ -0,0 +1,1123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 750,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.004,
+      "grad_norm": 0.515625,
+      "learning_rate": 4.993333333333333e-07,
+      "loss": 3.0113,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 0.443359375,
+      "learning_rate": 4.966666666666666e-07,
+      "loss": 3.1364,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 0.69921875,
+      "learning_rate": 4.933333333333333e-07,
+      "loss": 3.0859,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 0.419921875,
+      "learning_rate": 4.9e-07,
+      "loss": 2.7779,
+      "step": 15
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 0.453125,
+      "learning_rate": 4.866666666666666e-07,
+      "loss": 3.1198,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.44140625,
+      "learning_rate": 4.833333333333333e-07,
+      "loss": 2.9446,
+      "step": 25
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 0.59765625,
+      "learning_rate": 4.8e-07,
+      "loss": 2.9396,
+      "step": 30
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 0.7890625,
+      "learning_rate": 4.7666666666666667e-07,
+      "loss": 3.032,
+      "step": 35
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 0.443359375,
+      "learning_rate": 4.733333333333333e-07,
+      "loss": 2.9203,
+      "step": 40
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 0.462890625,
+      "learning_rate": 4.6999999999999995e-07,
+      "loss": 3.2915,
+      "step": 45
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 0.5390625,
+      "learning_rate": 4.6666666666666666e-07,
+      "loss": 3.3207,
+      "step": 50
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 0.71484375,
+      "learning_rate": 4.633333333333333e-07,
+      "loss": 2.9583,
+      "step": 55
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 0.48046875,
+      "learning_rate": 4.6e-07,
+      "loss": 3.133,
+      "step": 60
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 0.71875,
+      "learning_rate": 4.5666666666666665e-07,
+      "loss": 2.9234,
+      "step": 65
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 0.39453125,
+      "learning_rate": 4.5333333333333326e-07,
+      "loss": 2.9115,
+      "step": 70
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.5e-07,
+      "loss": 3.2351,
+      "step": 75
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 0.4765625,
+      "learning_rate": 4.4666666666666664e-07,
+      "loss": 2.9413,
+      "step": 80
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 0.44921875,
+      "learning_rate": 4.4333333333333336e-07,
+      "loss": 3.0558,
+      "step": 85
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 0.46484375,
+      "learning_rate": 4.3999999999999997e-07,
+      "loss": 2.9375,
+      "step": 90
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 0.5,
+      "learning_rate": 4.3666666666666663e-07,
+      "loss": 2.9765,
+      "step": 95
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.76953125,
+      "learning_rate": 4.3333333333333335e-07,
+      "loss": 2.8949,
+      "step": 100
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 0.455078125,
+      "learning_rate": 4.2999999999999996e-07,
+      "loss": 2.9066,
+      "step": 105
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 0.45703125,
+      "learning_rate": 4.266666666666667e-07,
+      "loss": 2.9894,
+      "step": 110
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 0.3515625,
+      "learning_rate": 4.2333333333333334e-07,
+      "loss": 3.0943,
+      "step": 115
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 0.6171875,
+      "learning_rate": 4.1999999999999995e-07,
+      "loss": 2.938,
+      "step": 120
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 0.59375,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 3.1882,
+      "step": 125
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 0.6015625,
+      "learning_rate": 4.1333333333333333e-07,
+      "loss": 3.0927,
+      "step": 130
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 0.427734375,
+      "learning_rate": 4.0999999999999994e-07,
+      "loss": 3.0316,
+      "step": 135
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 0.455078125,
+      "learning_rate": 4.0666666666666666e-07,
+      "loss": 3.0778,
+      "step": 140
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.033333333333333e-07,
+      "loss": 3.1895,
+      "step": 145
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 0.65625,
+      "learning_rate": 4e-07,
+      "loss": 2.8721,
+      "step": 150
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 0.609375,
+      "learning_rate": 3.9666666666666665e-07,
+      "loss": 3.1312,
+      "step": 155
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.6640625,
+      "learning_rate": 3.933333333333333e-07,
+      "loss": 3.1511,
+      "step": 160
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 0.54296875,
+      "learning_rate": 3.8999999999999997e-07,
+      "loss": 3.1,
+      "step": 165
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 0.412109375,
+      "learning_rate": 3.8666666666666664e-07,
+      "loss": 3.042,
+      "step": 170
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 1.0546875,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 2.9667,
+      "step": 175
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 0.37109375,
+      "learning_rate": 3.7999999999999996e-07,
+      "loss": 3.389,
+      "step": 180
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.766666666666666e-07,
+      "loss": 3.1293,
+      "step": 185
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 0.56640625,
+      "learning_rate": 3.7333333333333334e-07,
+      "loss": 3.0617,
+      "step": 190
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 0.62890625,
+      "learning_rate": 3.7e-07,
+      "loss": 3.1602,
+      "step": 195
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.54296875,
+      "learning_rate": 3.666666666666666e-07,
+      "loss": 3.0768,
+      "step": 200
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 0.4921875,
+      "learning_rate": 3.6333333333333333e-07,
+      "loss": 2.9545,
+      "step": 205
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 0.70703125,
+      "learning_rate": 3.6e-07,
+      "loss": 3.1713,
+      "step": 210
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 0.447265625,
+      "learning_rate": 3.5666666666666666e-07,
+      "loss": 3.0322,
+      "step": 215
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 0.498046875,
+      "learning_rate": 3.533333333333333e-07,
+      "loss": 2.9385,
+      "step": 220
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 0.59765625,
+      "learning_rate": 3.5e-07,
+      "loss": 3.0316,
+      "step": 225
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 0.3046875,
+      "learning_rate": 3.4666666666666665e-07,
+      "loss": 3.4933,
+      "step": 230
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 0.58984375,
+      "learning_rate": 3.433333333333333e-07,
+      "loss": 3.07,
+      "step": 235
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 0.71484375,
+      "learning_rate": 3.4000000000000003e-07,
+      "loss": 2.9426,
+      "step": 240
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 0.64453125,
+      "learning_rate": 3.3666666666666664e-07,
+      "loss": 3.122,
+      "step": 245
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.53515625,
+      "learning_rate": 3.333333333333333e-07,
+      "loss": 2.8377,
+      "step": 250
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 3.0472724437713623,
+      "eval_runtime": 37.1649,
+      "eval_samples_per_second": 2.691,
+      "eval_steps_per_second": 1.345,
+      "step": 250
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 0.4375,
+      "learning_rate": 3.3e-07,
+      "loss": 2.9272,
+      "step": 255
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 0.578125,
+      "learning_rate": 3.2666666666666663e-07,
+      "loss": 2.9946,
+      "step": 260
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.233333333333333e-07,
+      "loss": 3.3259,
+      "step": 265
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 0.609375,
+      "learning_rate": 3.2e-07,
+      "loss": 2.8449,
+      "step": 270
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 0.3203125,
+      "learning_rate": 3.166666666666666e-07,
+      "loss": 3.0095,
+      "step": 275
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 0.328125,
+      "learning_rate": 3.1333333333333333e-07,
+      "loss": 2.8402,
+      "step": 280
+    },
+    {
+      "epoch": 1.1400000000000001,
+      "grad_norm": 0.8984375,
+      "learning_rate": 3.1e-07,
+      "loss": 3.1718,
+      "step": 285
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 0.60546875,
+      "learning_rate": 3.066666666666666e-07,
+      "loss": 2.9385,
+      "step": 290
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 0.83203125,
+      "learning_rate": 3.033333333333333e-07,
+      "loss": 3.2249,
+      "step": 295
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.29296875,
+      "learning_rate": 3e-07,
+      "loss": 3.1023,
+      "step": 300
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 0.48828125,
+      "learning_rate": 2.966666666666667e-07,
+      "loss": 2.8704,
+      "step": 305
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 0.453125,
+      "learning_rate": 2.933333333333333e-07,
+      "loss": 3.1789,
+      "step": 310
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 0.455078125,
+      "learning_rate": 2.9e-07,
+      "loss": 3.135,
+      "step": 315
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 0.4609375,
+      "learning_rate": 2.866666666666667e-07,
+      "loss": 2.9556,
+      "step": 320
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 0.482421875,
+      "learning_rate": 2.833333333333333e-07,
+      "loss": 3.0085,
+      "step": 325
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 0.609375,
+      "learning_rate": 2.8e-07,
+      "loss": 3.1164,
+      "step": 330
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 0.48046875,
+      "learning_rate": 2.766666666666667e-07,
+      "loss": 3.137,
+      "step": 335
+    },
+    {
+      "epoch": 1.3599999999999999,
+      "grad_norm": 0.73828125,
+      "learning_rate": 2.733333333333333e-07,
+      "loss": 3.0196,
+      "step": 340
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 0.640625,
+      "learning_rate": 2.7e-07,
+      "loss": 3.1515,
+      "step": 345
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 0.400390625,
+      "learning_rate": 2.6666666666666667e-07,
+      "loss": 3.181,
+      "step": 350
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 0.462890625,
+      "learning_rate": 2.633333333333333e-07,
+      "loss": 3.0423,
+      "step": 355
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 0.56640625,
+      "learning_rate": 2.6e-07,
+      "loss": 3.2057,
+      "step": 360
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 0.416015625,
+      "learning_rate": 2.5666666666666666e-07,
+      "loss": 2.9651,
+      "step": 365
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 0.77734375,
+      "learning_rate": 2.533333333333333e-07,
+      "loss": 3.0129,
+      "step": 370
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 0.51953125,
+      "learning_rate": 2.5e-07,
+      "loss": 2.9392,
+      "step": 375
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 0.8125,
+      "learning_rate": 2.4666666666666665e-07,
+      "loss": 3.2183,
+      "step": 380
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 0.64453125,
+      "learning_rate": 2.433333333333333e-07,
+      "loss": 2.8756,
+      "step": 385
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 0.54296875,
+      "learning_rate": 2.4e-07,
+      "loss": 3.0185,
+      "step": 390
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 0.359375,
+      "learning_rate": 2.3666666666666664e-07,
+      "loss": 2.8795,
+      "step": 395
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.65625,
+      "learning_rate": 2.3333333333333333e-07,
+      "loss": 3.1054,
+      "step": 400
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 0.455078125,
+      "learning_rate": 2.3e-07,
+      "loss": 3.039,
+      "step": 405
+    },
+    {
+      "epoch": 1.6400000000000001,
+      "grad_norm": 0.76953125,
+      "learning_rate": 2.2666666666666663e-07,
+      "loss": 3.114,
+      "step": 410
+    },
+    {
+      "epoch": 1.6600000000000001,
+      "grad_norm": 0.7109375,
+      "learning_rate": 2.2333333333333332e-07,
+      "loss": 3.0176,
+      "step": 415
+    },
+    {
+      "epoch": 1.6800000000000002,
+      "grad_norm": 0.48046875,
+      "learning_rate": 2.1999999999999998e-07,
+      "loss": 3.014,
+      "step": 420
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 0.54296875,
+      "learning_rate": 2.1666666666666667e-07,
+      "loss": 2.9937,
+      "step": 425
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 0.474609375,
+      "learning_rate": 2.1333333333333334e-07,
+      "loss": 3.0268,
+      "step": 430
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 0.72265625,
+      "learning_rate": 2.0999999999999997e-07,
+      "loss": 3.1097,
+      "step": 435
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 0.5703125,
+      "learning_rate": 2.0666666666666666e-07,
+      "loss": 3.0335,
+      "step": 440
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 0.79296875,
+      "learning_rate": 2.0333333333333333e-07,
+      "loss": 3.0041,
+      "step": 445
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 0.59375,
+      "learning_rate": 2e-07,
+      "loss": 2.8949,
+      "step": 450
+    },
+    {
+      "epoch": 1.8199999999999998,
+      "grad_norm": 0.482421875,
+      "learning_rate": 1.9666666666666665e-07,
+      "loss": 3.1279,
+      "step": 455
+    },
+    {
+      "epoch": 1.8399999999999999,
+      "grad_norm": 0.61328125,
+      "learning_rate": 1.9333333333333332e-07,
+      "loss": 2.9593,
+      "step": 460
+    },
+    {
+      "epoch": 1.8599999999999999,
+      "grad_norm": 0.4609375,
+      "learning_rate": 1.8999999999999998e-07,
+      "loss": 2.9162,
+      "step": 465
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 0.5703125,
+      "learning_rate": 1.8666666666666667e-07,
+      "loss": 3.1065,
+      "step": 470
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 0.46875,
+      "learning_rate": 1.833333333333333e-07,
+      "loss": 3.2712,
+      "step": 475
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 0.48046875,
+      "learning_rate": 1.8e-07,
+      "loss": 2.988,
+      "step": 480
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 0.51171875,
+      "learning_rate": 1.7666666666666666e-07,
+      "loss": 3.0023,
+      "step": 485
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 0.6484375,
+      "learning_rate": 1.7333333333333332e-07,
+      "loss": 3.0329,
+      "step": 490
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 0.4765625,
+      "learning_rate": 1.7000000000000001e-07,
+      "loss": 2.8625,
+      "step": 495
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.63671875,
+      "learning_rate": 1.6666666666666665e-07,
+      "loss": 2.9821,
+      "step": 500
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 3.0425307750701904,
+      "eval_runtime": 37.1787,
+      "eval_samples_per_second": 2.69,
+      "eval_steps_per_second": 1.345,
+      "step": 500
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 0.50390625,
+      "learning_rate": 1.6333333333333331e-07,
+      "loss": 3.0855,
+      "step": 505
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 0.51953125,
+      "learning_rate": 1.6e-07,
+      "loss": 2.9503,
+      "step": 510
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 0.51171875,
+      "learning_rate": 1.5666666666666667e-07,
+      "loss": 2.9242,
+      "step": 515
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 0.52734375,
+      "learning_rate": 1.533333333333333e-07,
+      "loss": 3.1268,
+      "step": 520
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 0.439453125,
+      "learning_rate": 1.5e-07,
+      "loss": 3.0643,
+      "step": 525
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 0.53125,
+      "learning_rate": 1.4666666666666666e-07,
+      "loss": 3.0038,
+      "step": 530
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 0.46484375,
+      "learning_rate": 1.4333333333333335e-07,
+      "loss": 3.2325,
+      "step": 535
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 0.427734375,
+      "learning_rate": 1.4e-07,
+      "loss": 2.7652,
+      "step": 540
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 0.5625,
+      "learning_rate": 1.3666666666666665e-07,
+      "loss": 3.2086,
+      "step": 545
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 0.58203125,
+      "learning_rate": 1.3333333333333334e-07,
+      "loss": 3.0982,
+      "step": 550
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 0.474609375,
+      "learning_rate": 1.3e-07,
+      "loss": 3.1022,
+      "step": 555
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 0.41015625,
+      "learning_rate": 1.2666666666666666e-07,
+      "loss": 2.9948,
+      "step": 560
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.2333333333333333e-07,
+      "loss": 3.0617,
+      "step": 565
+    },
+    {
+      "epoch": 2.2800000000000002,
+      "grad_norm": 0.416015625,
+      "learning_rate": 1.2e-07,
+      "loss": 2.9632,
+      "step": 570
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 0.55078125,
+      "learning_rate": 1.1666666666666667e-07,
+      "loss": 3.0321,
+      "step": 575
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 0.44140625,
+      "learning_rate": 1.1333333333333332e-07,
+      "loss": 2.9494,
+      "step": 580
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 0.56640625,
+      "learning_rate": 1.0999999999999999e-07,
+      "loss": 2.9734,
+      "step": 585
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 0.423828125,
+      "learning_rate": 1.0666666666666667e-07,
+      "loss": 2.9476,
+      "step": 590
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 0.408203125,
+      "learning_rate": 1.0333333333333333e-07,
+      "loss": 2.9636,
+      "step": 595
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.353515625,
+      "learning_rate": 1e-07,
+      "loss": 2.9811,
+      "step": 600
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 0.58203125,
+      "learning_rate": 9.666666666666666e-08,
+      "loss": 3.1042,
+      "step": 605
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 0.984375,
+      "learning_rate": 9.333333333333334e-08,
+      "loss": 3.2165,
+      "step": 610
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 0.51171875,
+      "learning_rate": 9e-08,
+      "loss": 3.0101,
+      "step": 615
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 0.95703125,
+      "learning_rate": 8.666666666666666e-08,
+      "loss": 3.2647,
+      "step": 620
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 0.64453125,
+      "learning_rate": 8.333333333333333e-08,
+      "loss": 3.0509,
+      "step": 625
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 0.55859375,
+      "learning_rate": 8e-08,
+      "loss": 3.1422,
+      "step": 630
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 0.84765625,
+      "learning_rate": 7.666666666666665e-08,
+      "loss": 3.0169,
+      "step": 635
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.57421875,
+      "learning_rate": 7.333333333333333e-08,
+      "loss": 3.0908,
+      "step": 640
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 0.80078125,
+      "learning_rate": 7e-08,
+      "loss": 3.0154,
+      "step": 645
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 0.625,
+      "learning_rate": 6.666666666666667e-08,
+      "loss": 2.9793,
+      "step": 650
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 0.55078125,
+      "learning_rate": 6.333333333333333e-08,
+      "loss": 3.2019,
+      "step": 655
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 0.63671875,
+      "learning_rate": 6e-08,
+      "loss": 3.2179,
+      "step": 660
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 0.5546875,
+      "learning_rate": 5.666666666666666e-08,
+      "loss": 2.9562,
+      "step": 665
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 0.7890625,
+      "learning_rate": 5.3333333333333334e-08,
+      "loss": 3.1532,
+      "step": 670
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 0.546875,
+      "learning_rate": 5e-08,
+      "loss": 2.8235,
+      "step": 675
+    },
+    {
+      "epoch": 2.7199999999999998,
+      "grad_norm": 0.703125,
+      "learning_rate": 4.666666666666667e-08,
+      "loss": 2.999,
+      "step": 680
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 0.396484375,
+      "learning_rate": 4.333333333333333e-08,
+      "loss": 3.1061,
+      "step": 685
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 0.427734375,
+      "learning_rate": 4e-08,
+      "loss": 3.0664,
+      "step": 690
+    },
+    {
+      "epoch": 2.7800000000000002,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.6666666666666664e-08,
+      "loss": 3.0183,
+      "step": 695
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.58984375,
+      "learning_rate": 3.3333333333333334e-08,
+      "loss": 2.9407,
+      "step": 700
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 0.6875,
+      "learning_rate": 3e-08,
+      "loss": 2.9318,
+      "step": 705
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 0.68359375,
+      "learning_rate": 2.6666666666666667e-08,
+      "loss": 3.0431,
+      "step": 710
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 0.74609375,
+      "learning_rate": 2.3333333333333334e-08,
+      "loss": 2.9694,
+      "step": 715
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 0.4453125,
+      "learning_rate": 2e-08,
+      "loss": 2.9251,
+      "step": 720
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 0.439453125,
+      "learning_rate": 1.6666666666666667e-08,
+      "loss": 3.0582,
+      "step": 725
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 0.4140625,
+      "learning_rate": 1.3333333333333334e-08,
+      "loss": 2.835,
+      "step": 730
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 0.44140625,
+      "learning_rate": 1e-08,
+      "loss": 2.9812,
+      "step": 735
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 0.55859375,
+      "learning_rate": 6.666666666666667e-09,
+      "loss": 3.3347,
+      "step": 740
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 0.421875,
+      "learning_rate": 3.3333333333333334e-09,
+      "loss": 3.1562,
+      "step": 745
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.0,
+      "loss": 2.9724,
+      "step": 750
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 3.042874813079834,
+      "eval_runtime": 37.1498,
+      "eval_samples_per_second": 2.692,
+      "eval_steps_per_second": 1.346,
+      "step": 750
+    },
+    {
+      "epoch": 3.0,
+      "step": 750,
+      "total_flos": 3617591393476608.0,
+      "train_loss": 3.0439649012883505,
+      "train_runtime": 1336.343,
+      "train_samples_per_second": 1.122,
+      "train_steps_per_second": 0.561
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3617591393476608.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2825a13056975925f8567d40e4278de2dba1d9e79fa168675208cf57dc9cdeaf
+size 5432