Training in progress, step 25, checkpoint

last-checkpoint/README.md

@@ -18,7 +18,6 @@ base_model: bn22/Mistral-7B-Instruct-v0.1-sharded
 
 
 - **Developed by:** [More Information Needed]
-- **Funded by [optional]:** [More Information Needed]
 - **Shared by [optional]:** [More Information Needed]
 - **Model type:** [More Information Needed]
 - **Language(s) (NLP):** [More Information Needed]
@@ -77,7 +76,7 @@ Use the code below to get started with the model.
 
 ### Training Data
 
-<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+<!-- This should link to a Data Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
 
 [More Information Needed]
 
@@ -108,7 +107,7 @@ Use the code below to get started with the model.
 
 #### Testing Data
 
-<!-- This should link to a Dataset Card if possible. -->
+<!-- This should link to a Data Card if possible. -->
 
 [More Information Needed]
 
@@ -212,7 +211,7 @@ The following `bitsandbytes` quantization config was used during training:
 - llm_int8_has_fp16_weight: True
 - bnb_4bit_quant_type: nf4
 - bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: float16
+- bnb_4bit_compute_dtype: bfloat16
 
 ### Framework versions
 

last-checkpoint/adapter_config.json

@@ -16,13 +16,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "gate_proj",
+    "v_proj",
+    "q_proj",
     "up_proj",
+    "gate_proj",
+    "k_proj",
     "down_proj",
-    "q_proj",
-    "v_proj",
-    "k_proj"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d91107f5c242c8df5342db2e311f2309392ecf0548153ba97e37db1ac8814530
+oid sha256:80fa3a57d9708cddd9142d3819011418aedaa524c0b53b7c786d849cb8e3c617
 size 83945296

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24377944ce0dd27b0697f70f0d52d926be551d7d662470a7bb5e641270e05441
+oid sha256:d0d237a9a2cb2bf345606e0f288a49c7998e790371004a83437ab48963c73264
 size 42545748

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79438b1731d80b13c3e856d3e6ada7b258fe0028bd55a30293421a89363cfff2
+oid sha256:5d0d220e3ffb9cbe7aa8b7ee81ee25441366727f842c08121cff70e1f43b7a50
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16ad2180f5b27e9af3551807b0da763fbb16c05640f8b967bf27f34341ae1380
+oid sha256:350b7a6f8159717ef2af74e68b6e9c8ed2775fab22debcfdf6f728bfa1bef4ad
 size 1064

last-checkpoint/trainer_state.json

@@ -1,619 +1,169 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 100.0,
+  "epoch": 0.005755064456721915,
   "eval_steps": 1000,
-  "global_step": 100,
+  "global_step": 25,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 1.0,
-      "learning_rate": 0.0,
-      "loss": 1.399,
-      "step": 1
-    },
-    {
-      "epoch": 2.0,
-      "learning_rate": 0.0,
-      "loss": 1.399,
-      "step": 2
-    },
-    {
-      "epoch": 3.0,
-      "learning_rate": 0.0,
-      "loss": 1.399,
-      "step": 3
-    },
-    {
-      "epoch": 4.0,
-      "learning_rate": 0.0,
-      "loss": 1.399,
-      "step": 4
-    },
-    {
-      "epoch": 5.0,
+      "epoch": 0.0,
       "learning_rate": 4e-05,
-      "loss": 1.399,
-      "step": 5
+      "loss": 3.6137,
+      "step": 1
     },
     {
-      "epoch": 6.0,
+      "epoch": 0.0,
       "learning_rate": 8e-05,
-      "loss": 1.399,
-      "step": 6
+      "loss": 3.4233,
+      "step": 2
     },
     {
-      "epoch": 7.0,
+      "epoch": 0.0,
       "learning_rate": 0.00012,
-      "loss": 1.2225,
-      "step": 7
+      "loss": 3.3927,
+      "step": 3
     },
     {
-      "epoch": 8.0,
+      "epoch": 0.0,
       "learning_rate": 0.00016,
-      "loss": 0.9757,
-      "step": 8
+      "loss": 2.8163,
+      "step": 4
     },
     {
-      "epoch": 9.0,
+      "epoch": 0.0,
       "learning_rate": 0.0002,
-      "loss": 0.6616,
-      "step": 9
+      "loss": 2.4858,
+      "step": 5
     },
     {
-      "epoch": 10.0,
+      "epoch": 0.0,
       "learning_rate": 0.00019789473684210526,
-      "loss": 0.34,
-      "step": 10
+      "loss": 1.9139,
+      "step": 6
     },
     {
-      "epoch": 11.0,
+      "epoch": 0.0,
       "learning_rate": 0.00019578947368421054,
-      "loss": 0.1851,
-      "step": 11
+      "loss": 2.0399,
+      "step": 7
     },
     {
-      "epoch": 12.0,
+      "epoch": 0.0,
       "learning_rate": 0.0001936842105263158,
-      "loss": 0.18,
-      "step": 12
+      "loss": 1.9175,
+      "step": 8
     },
     {
-      "epoch": 13.0,
+      "epoch": 0.0,
       "learning_rate": 0.00019157894736842104,
-      "loss": 0.1289,
-      "step": 13
+      "loss": 2.2095,
+      "step": 9
     },
     {
-      "epoch": 14.0,
+      "epoch": 0.0,
       "learning_rate": 0.00018947368421052632,
-      "loss": 0.0939,
-      "step": 14
+      "loss": 1.9044,
+      "step": 10
     },
     {
-      "epoch": 15.0,
+      "epoch": 0.0,
       "learning_rate": 0.0001873684210526316,
-      "loss": 0.0693,
-      "step": 15
+      "loss": 2.013,
+      "step": 11
     },
     {
-      "epoch": 16.0,
+      "epoch": 0.0,
       "learning_rate": 0.00018526315789473685,
-      "loss": 0.0616,
-      "step": 16
+      "loss": 2.1313,
+      "step": 12
     },
     {
-      "epoch": 17.0,
+      "epoch": 0.0,
       "learning_rate": 0.0001831578947368421,
-      "loss": 0.0507,
-      "step": 17
+      "loss": 1.9865,
+      "step": 13
     },
     {
-      "epoch": 18.0,
+      "epoch": 0.0,
       "learning_rate": 0.00018105263157894739,
-      "loss": 0.0416,
-      "step": 18
+      "loss": 1.7398,
+      "step": 14
     },
     {
-      "epoch": 19.0,
+      "epoch": 0.0,
       "learning_rate": 0.00017894736842105264,
-      "loss": 0.0413,
-      "step": 19
+      "loss": 1.693,
+      "step": 15
     },
     {
-      "epoch": 20.0,
+      "epoch": 0.0,
       "learning_rate": 0.0001768421052631579,
-      "loss": 0.0545,
-      "step": 20
+      "loss": 1.6588,
+      "step": 16
     },
     {
-      "epoch": 21.0,
+      "epoch": 0.0,
       "learning_rate": 0.00017473684210526317,
-      "loss": 0.0323,
-      "step": 21
+      "loss": 1.989,
+      "step": 17
     },
     {
-      "epoch": 22.0,
+      "epoch": 0.0,
       "learning_rate": 0.00017263157894736842,
-      "loss": 0.0254,
-      "step": 22
+      "loss": 1.9944,
+      "step": 18
     },
     {
-      "epoch": 23.0,
+      "epoch": 0.0,
       "learning_rate": 0.0001705263157894737,
-      "loss": 0.0052,
-      "step": 23
+      "loss": 1.7393,
+      "step": 19
     },
     {
-      "epoch": 24.0,
+      "epoch": 0.0,
       "learning_rate": 0.00016842105263157895,
-      "loss": 0.0047,
-      "step": 24
+      "loss": 1.7573,
+      "step": 20
     },
     {
-      "epoch": 25.0,
+      "epoch": 0.0,
       "learning_rate": 0.00016631578947368423,
-      "loss": 0.0127,
-      "step": 25
+      "loss": 1.546,
+      "step": 21
     },
     {
-      "epoch": 26.0,
+      "epoch": 0.01,
       "learning_rate": 0.00016421052631578948,
-      "loss": 0.0049,
-      "step": 26
+      "loss": 1.6286,
+      "step": 22
     },
     {
-      "epoch": 27.0,
+      "epoch": 0.01,
       "learning_rate": 0.00016210526315789473,
-      "loss": 0.0042,
-      "step": 27
+      "loss": 1.61,
+      "step": 23
     },
     {
-      "epoch": 28.0,
+      "epoch": 0.01,
       "learning_rate": 0.00016,
-      "loss": 0.0052,
-      "step": 28
+      "loss": 1.7942,
+      "step": 24
     },
     {
-      "epoch": 29.0,
+      "epoch": 0.01,
       "learning_rate": 0.00015789473684210527,
-      "loss": 0.0041,
-      "step": 29
-    },
-    {
-      "epoch": 30.0,
-      "learning_rate": 0.00015578947368421052,
-      "loss": 0.0044,
-      "step": 30
-    },
-    {
-      "epoch": 31.0,
-      "learning_rate": 0.0001536842105263158,
-      "loss": 0.0044,
-      "step": 31
-    },
-    {
-      "epoch": 32.0,
-      "learning_rate": 0.00015157894736842108,
-      "loss": 0.0043,
-      "step": 32
-    },
-    {
-      "epoch": 33.0,
-      "learning_rate": 0.00014947368421052633,
-      "loss": 0.0043,
-      "step": 33
-    },
-    {
-      "epoch": 34.0,
-      "learning_rate": 0.00014736842105263158,
-      "loss": 0.0043,
-      "step": 34
-    },
-    {
-      "epoch": 35.0,
-      "learning_rate": 0.00014526315789473686,
-      "loss": 0.0044,
-      "step": 35
-    },
-    {
-      "epoch": 36.0,
-      "learning_rate": 0.0001431578947368421,
-      "loss": 0.0035,
-      "step": 36
-    },
-    {
-      "epoch": 37.0,
-      "learning_rate": 0.00014105263157894736,
-      "loss": 0.004,
-      "step": 37
-    },
-    {
-      "epoch": 38.0,
-      "learning_rate": 0.00013894736842105264,
-      "loss": 0.0044,
-      "step": 38
-    },
-    {
-      "epoch": 39.0,
-      "learning_rate": 0.0001368421052631579,
-      "loss": 0.0041,
-      "step": 39
-    },
-    {
-      "epoch": 40.0,
-      "learning_rate": 0.00013473684210526317,
-      "loss": 0.0043,
-      "step": 40
-    },
-    {
-      "epoch": 41.0,
-      "learning_rate": 0.00013263157894736842,
-      "loss": 0.0041,
-      "step": 41
-    },
-    {
-      "epoch": 42.0,
-      "learning_rate": 0.0001305263157894737,
-      "loss": 0.0042,
-      "step": 42
-    },
-    {
-      "epoch": 43.0,
-      "learning_rate": 0.00012842105263157895,
-      "loss": 0.0036,
-      "step": 43
-    },
-    {
-      "epoch": 44.0,
-      "learning_rate": 0.0001263157894736842,
-      "loss": 0.0038,
-      "step": 44
-    },
-    {
-      "epoch": 45.0,
-      "learning_rate": 0.00012421052631578949,
-      "loss": 0.0039,
-      "step": 45
-    },
-    {
-      "epoch": 46.0,
-      "learning_rate": 0.00012210526315789474,
-      "loss": 0.0045,
-      "step": 46
-    },
-    {
-      "epoch": 47.0,
-      "learning_rate": 0.00012,
-      "loss": 0.0053,
-      "step": 47
-    },
-    {
-      "epoch": 48.0,
-      "learning_rate": 0.00011789473684210525,
-      "loss": 0.0042,
-      "step": 48
-    },
-    {
-      "epoch": 49.0,
-      "learning_rate": 0.00011578947368421053,
-      "loss": 0.0029,
-      "step": 49
-    },
-    {
-      "epoch": 50.0,
-      "learning_rate": 0.0001136842105263158,
-      "loss": 0.004,
-      "step": 50
-    },
-    {
-      "epoch": 51.0,
-      "learning_rate": 0.00011157894736842105,
-      "loss": 0.0042,
-      "step": 51
-    },
-    {
-      "epoch": 52.0,
-      "learning_rate": 0.00010947368421052633,
-      "loss": 0.0039,
-      "step": 52
-    },
-    {
-      "epoch": 53.0,
-      "learning_rate": 0.00010736842105263158,
-      "loss": 0.0046,
-      "step": 53
-    },
-    {
-      "epoch": 54.0,
-      "learning_rate": 0.00010526315789473685,
-      "loss": 0.0041,
-      "step": 54
-    },
-    {
-      "epoch": 55.0,
-      "learning_rate": 0.00010315789473684211,
-      "loss": 0.0042,
-      "step": 55
-    },
-    {
-      "epoch": 56.0,
-      "learning_rate": 0.00010105263157894738,
-      "loss": 0.0042,
-      "step": 56
-    },
-    {
-      "epoch": 57.0,
-      "learning_rate": 9.894736842105263e-05,
-      "loss": 0.0037,
-      "step": 57
-    },
-    {
-      "epoch": 58.0,
-      "learning_rate": 9.68421052631579e-05,
-      "loss": 0.0038,
-      "step": 58
-    },
-    {
-      "epoch": 59.0,
-      "learning_rate": 9.473684210526316e-05,
-      "loss": 0.0042,
-      "step": 59
-    },
-    {
-      "epoch": 60.0,
-      "learning_rate": 9.263157894736843e-05,
-      "loss": 0.0042,
-      "step": 60
-    },
-    {
-      "epoch": 61.0,
-      "learning_rate": 9.052631578947369e-05,
-      "loss": 0.0042,
-      "step": 61
-    },
-    {
-      "epoch": 62.0,
-      "learning_rate": 8.842105263157894e-05,
-      "loss": 0.0038,
-      "step": 62
-    },
-    {
-      "epoch": 63.0,
-      "learning_rate": 8.631578947368421e-05,
-      "loss": 0.0041,
-      "step": 63
-    },
-    {
-      "epoch": 64.0,
-      "learning_rate": 8.421052631578948e-05,
-      "loss": 0.0036,
-      "step": 64
-    },
-    {
-      "epoch": 65.0,
-      "learning_rate": 8.210526315789474e-05,
-      "loss": 0.0042,
-      "step": 65
-    },
-    {
-      "epoch": 66.0,
-      "learning_rate": 8e-05,
-      "loss": 0.004,
-      "step": 66
-    },
-    {
-      "epoch": 67.0,
-      "learning_rate": 7.789473684210526e-05,
-      "loss": 0.0036,
-      "step": 67
-    },
-    {
-      "epoch": 68.0,
-      "learning_rate": 7.578947368421054e-05,
-      "loss": 0.0043,
-      "step": 68
-    },
-    {
-      "epoch": 69.0,
-      "learning_rate": 7.368421052631579e-05,
-      "loss": 0.0037,
-      "step": 69
-    },
-    {
-      "epoch": 70.0,
-      "learning_rate": 7.157894736842105e-05,
-      "loss": 0.0038,
-      "step": 70
-    },
-    {
-      "epoch": 71.0,
-      "learning_rate": 6.947368421052632e-05,
-      "loss": 0.0043,
-      "step": 71
-    },
-    {
-      "epoch": 72.0,
-      "learning_rate": 6.736842105263159e-05,
-      "loss": 0.004,
-      "step": 72
-    },
-    {
-      "epoch": 73.0,
-      "learning_rate": 6.526315789473685e-05,
-      "loss": 0.0043,
-      "step": 73
-    },
-    {
-      "epoch": 74.0,
-      "learning_rate": 6.31578947368421e-05,
-      "loss": 0.0041,
-      "step": 74
-    },
-    {
-      "epoch": 75.0,
-      "learning_rate": 6.105263157894737e-05,
-      "loss": 0.0034,
-      "step": 75
-    },
-    {
-      "epoch": 76.0,
-      "learning_rate": 5.894736842105263e-05,
-      "loss": 0.0041,
-      "step": 76
-    },
-    {
-      "epoch": 77.0,
-      "learning_rate": 5.68421052631579e-05,
-      "loss": 0.0041,
-      "step": 77
-    },
-    {
-      "epoch": 78.0,
-      "learning_rate": 5.4736842105263165e-05,
-      "loss": 0.0034,
-      "step": 78
-    },
-    {
-      "epoch": 79.0,
-      "learning_rate": 5.2631578947368424e-05,
-      "loss": 0.0038,
-      "step": 79
-    },
-    {
-      "epoch": 80.0,
-      "learning_rate": 5.052631578947369e-05,
-      "loss": 0.0037,
-      "step": 80
-    },
-    {
-      "epoch": 81.0,
-      "learning_rate": 4.842105263157895e-05,
-      "loss": 0.0044,
-      "step": 81
-    },
-    {
-      "epoch": 82.0,
-      "learning_rate": 4.6315789473684214e-05,
-      "loss": 0.0044,
-      "step": 82
-    },
-    {
-      "epoch": 83.0,
-      "learning_rate": 4.421052631578947e-05,
-      "loss": 0.0042,
-      "step": 83
-    },
-    {
-      "epoch": 84.0,
-      "learning_rate": 4.210526315789474e-05,
-      "loss": 0.0037,
-      "step": 84
-    },
-    {
-      "epoch": 85.0,
-      "learning_rate": 4e-05,
-      "loss": 0.0039,
-      "step": 85
-    },
-    {
-      "epoch": 86.0,
-      "learning_rate": 3.789473684210527e-05,
-      "loss": 0.0029,
-      "step": 86
-    },
-    {
-      "epoch": 87.0,
-      "learning_rate": 3.578947368421053e-05,
-      "loss": 0.0039,
-      "step": 87
-    },
-    {
-      "epoch": 88.0,
-      "learning_rate": 3.368421052631579e-05,
-      "loss": 0.004,
-      "step": 88
-    },
-    {
-      "epoch": 89.0,
-      "learning_rate": 3.157894736842105e-05,
-      "loss": 0.0042,
-      "step": 89
-    },
-    {
-      "epoch": 90.0,
-      "learning_rate": 2.9473684210526314e-05,
-      "loss": 0.0038,
-      "step": 90
-    },
-    {
-      "epoch": 91.0,
-      "learning_rate": 2.7368421052631583e-05,
-      "loss": 0.0035,
-      "step": 91
-    },
-    {
-      "epoch": 92.0,
-      "learning_rate": 2.5263157894736845e-05,
-      "loss": 0.0041,
-      "step": 92
-    },
-    {
-      "epoch": 93.0,
-      "learning_rate": 2.3157894736842107e-05,
-      "loss": 0.0039,
-      "step": 93
-    },
-    {
-      "epoch": 94.0,
-      "learning_rate": 2.105263157894737e-05,
-      "loss": 0.0034,
-      "step": 94
-    },
-    {
-      "epoch": 95.0,
-      "learning_rate": 1.8947368421052634e-05,
-      "loss": 0.0044,
-      "step": 95
-    },
-    {
-      "epoch": 96.0,
-      "learning_rate": 1.6842105263157896e-05,
-      "loss": 0.0041,
-      "step": 96
-    },
-    {
-      "epoch": 97.0,
-      "learning_rate": 1.4736842105263157e-05,
-      "loss": 0.0036,
-      "step": 97
-    },
-    {
-      "epoch": 98.0,
-      "learning_rate": 1.2631578947368422e-05,
-      "loss": 0.0037,
-      "step": 98
-    },
-    {
-      "epoch": 99.0,
-      "learning_rate": 1.0526315789473684e-05,
-      "loss": 0.006,
-      "step": 99
-    },
-    {
-      "epoch": 100.0,
-      "learning_rate": 8.421052631578948e-06,
-      "loss": 0.0041,
-      "step": 100
+      "loss": 1.4316,
+      "step": 25
     }
   ],
   "logging_steps": 1,
   "max_steps": 100,
-  "num_train_epochs": 100,
+  "num_train_epochs": 1,
   "save_steps": 25,
-  "total_flos": 1069744742400000.0,
+  "total_flos": 505946473365504.0,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b905f14608904b19068aed8f635b4021ef5a4d4a28a5ffd1e2cb0eebda395135
-size 4600
+oid sha256:e140806e2e2e31fba82f540cf5fc839f41cb8ed4903678e59fb42ac85a482011
+size 4536