Model save

README.md

@@ -0,0 +1,82 @@
+---
+base_model: meta-llama/Meta-Llama-3-8B
+library_name: peft
+license: llama3
+metrics:
+- accuracy
+tags:
+- generated_from_trainer
+model-index:
+- name: Output_llama2_70-15-15
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Output_llama2_70-15-15
+
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6250
+- Balanced Accuracy: 0.6326
+- Accuracy: 0.6282
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 16
+- eval_batch_size: 16
+- seed: 42
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 20
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Balanced Accuracy | Accuracy |
+|:-------------:|:-----:|:----:|:---------------:|:-----------------:|:--------:|
+| No log        | 1.0   | 46   | 0.7111          | 0.5764            | 0.5641   |
+| No log        | 2.0   | 92   | 0.7043          | 0.5656            | 0.5577   |
+| No log        | 3.0   | 138  | 0.6619          | 0.5142            | 0.5192   |
+| No log        | 4.0   | 184  | 0.7013          | 0.5595            | 0.5513   |
+| No log        | 5.0   | 230  | 0.6493          | 0.5620            | 0.5577   |
+| No log        | 6.0   | 276  | 0.6496          | 0.5671            | 0.5641   |
+| No log        | 7.0   | 322  | 0.6466          | 0.5798            | 0.5769   |
+| No log        | 8.0   | 368  | 0.6748          | 0.5527            | 0.5513   |
+| No log        | 9.0   | 414  | 0.6551          | 0.5692            | 0.5705   |
+| No log        | 10.0  | 460  | 0.6205          | 0.6063            | 0.5833   |
+| 0.6541        | 11.0  | 506  | 0.6537          | 0.6020            | 0.6026   |
+| 0.6541        | 12.0  | 552  | 0.6379          | 0.6167            | 0.6154   |
+| 0.6541        | 13.0  | 598  | 0.6243          | 0.6107            | 0.6026   |
+| 0.6541        | 14.0  | 644  | 0.6248          | 0.6074            | 0.6026   |
+| 0.6541        | 15.0  | 690  | 0.6172          | 0.6370            | 0.6218   |
+| 0.6541        | 16.0  | 736  | 0.6237          | 0.6202            | 0.6154   |
+| 0.6541        | 17.0  | 782  | 0.6308          | 0.6230            | 0.6218   |
+| 0.6541        | 18.0  | 828  | 0.6179          | 0.6319            | 0.6218   |
+| 0.6541        | 19.0  | 874  | 0.6252          | 0.6326            | 0.6282   |
+| 0.6541        | 20.0  | 920  | 0.6250          | 0.6326            | 0.6282   |
+
+
+### Framework versions
+
+- PEFT 0.10.0
+- Transformers 4.41.2
+- Pytorch 2.3.1+cu121
+- Datasets 3.2.0
+- Tokenizers 0.19.1

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae0d06c0f3b3a658f6f5f6b97c0c94a7d715e8b0954890d1fff0342ba1faf3fc
+oid sha256:5eb33dc0f3ed1883b5b290afe98054c3bb46d9c133c067783affba82d91a06f0
 size 16818656

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 20.0,
-    "total_flos": 4.976764349571072e+17,
-    "train_loss": 0.4557167053222656,
-    "train_runtime": 45521.9577,
+    "total_flos": 4.970048339828736e+17,
+    "train_loss": 0.6203255363132643,
+    "train_runtime": 33373.6734,
     "train_samples": 730,
-    "train_samples_per_second": 0.321,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.437,
+    "train_steps_per_second": 0.028
 }

tokenizer.json

@@ -6,7 +6,14 @@
     "strategy": "LongestFirst",
     "stride": 0
   },
-  "padding": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 128001,
+    "pad_type_id": 0,
+    "pad_token": "<|end_of_text|>"
+  },
   "added_tokens": [
     {
       "id": 128000,

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 20.0,
-    "total_flos": 4.976764349571072e+17,
-    "train_loss": 0.4557167053222656,
-    "train_runtime": 45521.9577,
+    "total_flos": 4.970048339828736e+17,
+    "train_loss": 0.6203255363132643,
+    "train_runtime": 33373.6734,
     "train_samples": 730,
-    "train_samples_per_second": 0.321,
-    "train_steps_per_second": 0.02
+    "train_samples_per_second": 0.437,
+    "train_steps_per_second": 0.028
 }

trainer_state.json

@@ -1,6 +1,6 @@
 {
-  "best_metric": 0.7435897435897436,
-  "best_model_checkpoint": "Output_llama2_70-15-15/checkpoint-460",
+  "best_metric": 0.6282051282051282,
+  "best_model_checkpoint": "Output_llama2_70-15-15/checkpoint-874",
   "epoch": 20.0,
   "eval_steps": 500,
   "global_step": 920,
@@ -10,219 +10,219 @@
   "log_history": [
     {
       "epoch": 1.0,
-      "eval_accuracy": 0.6538461538461539,
-      "eval_balanced_accuracy": 0.654945054945055,
-      "eval_loss": 0.6548082232475281,
-      "eval_runtime": 142.4927,
-      "eval_samples_per_second": 1.095,
-      "eval_steps_per_second": 0.07,
+      "eval_accuracy": 0.5641025641025641,
+      "eval_balanced_accuracy": 0.5764176417641764,
+      "eval_loss": 0.7111307978630066,
+      "eval_runtime": 111.2436,
+      "eval_samples_per_second": 1.402,
+      "eval_steps_per_second": 0.09,
       "step": 46
     },
     {
       "epoch": 2.0,
-      "eval_accuracy": 0.6153846153846154,
-      "eval_balanced_accuracy": 0.6180613090306546,
-      "eval_loss": 0.659588634967804,
-      "eval_runtime": 174.4587,
-      "eval_samples_per_second": 0.894,
-      "eval_steps_per_second": 0.057,
+      "eval_accuracy": 0.5576923076923077,
+      "eval_balanced_accuracy": 0.565625,
+      "eval_loss": 0.7042574882507324,
+      "eval_runtime": 111.3987,
+      "eval_samples_per_second": 1.4,
+      "eval_steps_per_second": 0.09,
       "step": 92
     },
     {
       "epoch": 3.0,
-      "eval_accuracy": 0.5833333333333334,
-      "eval_balanced_accuracy": 0.7773972602739726,
-      "eval_loss": 0.5924286842346191,
-      "eval_runtime": 145.3834,
-      "eval_samples_per_second": 1.073,
-      "eval_steps_per_second": 0.069,
+      "eval_accuracy": 0.5192307692307693,
+      "eval_balanced_accuracy": 0.5141509433962264,
+      "eval_loss": 0.6619334816932678,
+      "eval_runtime": 111.2028,
+      "eval_samples_per_second": 1.403,
+      "eval_steps_per_second": 0.09,
       "step": 138
     },
     {
       "epoch": 4.0,
-      "eval_accuracy": 0.6474358974358975,
-      "eval_balanced_accuracy": 0.6468253968253967,
-      "eval_loss": 0.621242105960846,
-      "eval_runtime": 157.0532,
-      "eval_samples_per_second": 0.993,
-      "eval_steps_per_second": 0.064,
+      "eval_accuracy": 0.5512820512820513,
+      "eval_balanced_accuracy": 0.5594967674296698,
+      "eval_loss": 0.7012639045715332,
+      "eval_runtime": 111.1032,
+      "eval_samples_per_second": 1.404,
+      "eval_steps_per_second": 0.09,
       "step": 184
     },
     {
       "epoch": 5.0,
-      "eval_accuracy": 0.6666666666666666,
-      "eval_balanced_accuracy": 0.6741095162147794,
-      "eval_loss": 0.5860592722892761,
-      "eval_runtime": 152.4923,
-      "eval_samples_per_second": 1.023,
-      "eval_steps_per_second": 0.066,
+      "eval_accuracy": 0.5576923076923077,
+      "eval_balanced_accuracy": 0.5620300751879699,
+      "eval_loss": 0.6493321657180786,
+      "eval_runtime": 111.1592,
+      "eval_samples_per_second": 1.403,
+      "eval_steps_per_second": 0.09,
       "step": 230
     },
     {
       "epoch": 6.0,
-      "eval_accuracy": 0.6538461538461539,
-      "eval_balanced_accuracy": 0.6597350327016602,
-      "eval_loss": 0.6232044696807861,
-      "eval_runtime": 143.706,
-      "eval_samples_per_second": 1.086,
-      "eval_steps_per_second": 0.07,
+      "eval_accuracy": 0.5641025641025641,
+      "eval_balanced_accuracy": 0.5670505563146594,
+      "eval_loss": 0.6495689749717712,
+      "eval_runtime": 111.1491,
+      "eval_samples_per_second": 1.404,
+      "eval_steps_per_second": 0.09,
       "step": 276
     },
     {
       "epoch": 7.0,
-      "eval_accuracy": 0.6346153846153846,
-      "eval_balanced_accuracy": 0.6890120967741935,
-      "eval_loss": 0.5387519001960754,
-      "eval_runtime": 153.5233,
-      "eval_samples_per_second": 1.016,
-      "eval_steps_per_second": 0.065,
+      "eval_accuracy": 0.5769230769230769,
+      "eval_balanced_accuracy": 0.5798319327731092,
+      "eval_loss": 0.6466049551963806,
+      "eval_runtime": 111.1355,
+      "eval_samples_per_second": 1.404,
+      "eval_steps_per_second": 0.09,
       "step": 322
     },
     {
       "epoch": 8.0,
-      "eval_accuracy": 0.6923076923076923,
-      "eval_balanced_accuracy": 0.7074555779446785,
-      "eval_loss": 0.5306457281112671,
-      "eval_runtime": 158.9271,
-      "eval_samples_per_second": 0.982,
-      "eval_steps_per_second": 0.063,
+      "eval_accuracy": 0.5512820512820513,
+      "eval_balanced_accuracy": 0.5527314738405678,
+      "eval_loss": 0.6747537851333618,
+      "eval_runtime": 111.3484,
+      "eval_samples_per_second": 1.401,
+      "eval_steps_per_second": 0.09,
       "step": 368
     },
     {
       "epoch": 9.0,
-      "eval_accuracy": 0.6923076923076923,
-      "eval_balanced_accuracy": 0.6994583260527695,
-      "eval_loss": 0.5140182375907898,
-      "eval_runtime": 145.2382,
-      "eval_samples_per_second": 1.074,
-      "eval_steps_per_second": 0.069,
+      "eval_accuracy": 0.5705128205128205,
+      "eval_balanced_accuracy": 0.5691844919786097,
+      "eval_loss": 0.6550834774971008,
+      "eval_runtime": 111.4062,
+      "eval_samples_per_second": 1.4,
+      "eval_steps_per_second": 0.09,
       "step": 414
     },
     {
       "epoch": 10.0,
-      "eval_accuracy": 0.7435897435897436,
-      "eval_balanced_accuracy": 0.7438276719138359,
-      "eval_loss": 0.513781726360321,
-      "eval_runtime": 144.459,
-      "eval_samples_per_second": 1.08,
-      "eval_steps_per_second": 0.069,
+      "eval_accuracy": 0.5833333333333334,
+      "eval_balanced_accuracy": 0.6063162970106075,
+      "eval_loss": 0.6205254793167114,
+      "eval_runtime": 111.3582,
+      "eval_samples_per_second": 1.401,
+      "eval_steps_per_second": 0.09,
       "step": 460
     },
     {
       "epoch": 10.869565217391305,
-      "grad_norm": 22.628210067749023,
+      "grad_norm": 3.162184715270996,
       "learning_rate": 4.565217391304348e-06,
-      "loss": 0.5585,
+      "loss": 0.6541,
       "step": 500
     },
     {
       "epoch": 11.0,
-      "eval_accuracy": 0.7371794871794872,
-      "eval_balanced_accuracy": 0.7398989898989898,
-      "eval_loss": 0.49725937843322754,
-      "eval_runtime": 145.8734,
-      "eval_samples_per_second": 1.069,
-      "eval_steps_per_second": 0.069,
+      "eval_accuracy": 0.6025641025641025,
+      "eval_balanced_accuracy": 0.6019753086419752,
+      "eval_loss": 0.6536551117897034,
+      "eval_runtime": 111.1486,
+      "eval_samples_per_second": 1.404,
+      "eval_steps_per_second": 0.09,
       "step": 506
     },
     {
       "epoch": 12.0,
-      "eval_accuracy": 0.6794871794871795,
-      "eval_balanced_accuracy": 0.6999809269502193,
-      "eval_loss": 0.49190282821655273,
-      "eval_runtime": 148.1722,
-      "eval_samples_per_second": 1.053,
-      "eval_steps_per_second": 0.067,
+      "eval_accuracy": 0.6153846153846154,
+      "eval_balanced_accuracy": 0.6167385677308024,
+      "eval_loss": 0.6378962397575378,
+      "eval_runtime": 111.1235,
+      "eval_samples_per_second": 1.404,
+      "eval_steps_per_second": 0.09,
       "step": 552
     },
     {
       "epoch": 13.0,
-      "eval_accuracy": 0.717948717948718,
-      "eval_balanced_accuracy": 0.7293992557150452,
-      "eval_loss": 0.4932830035686493,
-      "eval_runtime": 150.2623,
-      "eval_samples_per_second": 1.038,
-      "eval_steps_per_second": 0.067,
+      "eval_accuracy": 0.6025641025641025,
+      "eval_balanced_accuracy": 0.6107190539767309,
+      "eval_loss": 0.6242974400520325,
+      "eval_runtime": 111.2048,
+      "eval_samples_per_second": 1.403,
+      "eval_steps_per_second": 0.09,
       "step": 598
     },
     {
       "epoch": 14.0,
-      "eval_accuracy": 0.7051282051282052,
-      "eval_balanced_accuracy": 0.7297348846080488,
-      "eval_loss": 0.49414268136024475,
-      "eval_runtime": 152.7381,
-      "eval_samples_per_second": 1.021,
-      "eval_steps_per_second": 0.065,
+      "eval_accuracy": 0.6025641025641025,
+      "eval_balanced_accuracy": 0.60743725957135,
+      "eval_loss": 0.6248365640640259,
+      "eval_runtime": 111.2066,
+      "eval_samples_per_second": 1.403,
+      "eval_steps_per_second": 0.09,
       "step": 644
     },
     {
       "epoch": 15.0,
-      "eval_accuracy": 0.7115384615384616,
-      "eval_balanced_accuracy": 0.7239285714285715,
-      "eval_loss": 0.4975796341896057,
-      "eval_runtime": 148.3264,
-      "eval_samples_per_second": 1.052,
-      "eval_steps_per_second": 0.067,
+      "eval_accuracy": 0.6217948717948718,
+      "eval_balanced_accuracy": 0.6369565217391304,
+      "eval_loss": 0.6171802878379822,
+      "eval_runtime": 111.2635,
+      "eval_samples_per_second": 1.402,
+      "eval_steps_per_second": 0.09,
       "step": 690
     },
     {
       "epoch": 16.0,
-      "eval_accuracy": 0.7371794871794872,
-      "eval_balanced_accuracy": 0.7386363636363636,
-      "eval_loss": 0.5156741738319397,
-      "eval_runtime": 151.5711,
-      "eval_samples_per_second": 1.029,
-      "eval_steps_per_second": 0.066,
+      "eval_accuracy": 0.6153846153846154,
+      "eval_balanced_accuracy": 0.6201620162016201,
+      "eval_loss": 0.6236566305160522,
+      "eval_runtime": 111.2833,
+      "eval_samples_per_second": 1.402,
+      "eval_steps_per_second": 0.09,
       "step": 736
     },
     {
       "epoch": 17.0,
-      "eval_accuracy": 0.717948717948718,
-      "eval_balanced_accuracy": 0.720879120879121,
-      "eval_loss": 0.5169395208358765,
-      "eval_runtime": 152.1754,
-      "eval_samples_per_second": 1.025,
-      "eval_steps_per_second": 0.066,
+      "eval_accuracy": 0.6217948717948718,
+      "eval_balanced_accuracy": 0.6230267673301304,
+      "eval_loss": 0.630794107913971,
+      "eval_runtime": 111.1945,
+      "eval_samples_per_second": 1.403,
+      "eval_steps_per_second": 0.09,
       "step": 782
     },
     {
       "epoch": 18.0,
-      "eval_accuracy": 0.7243589743589743,
-      "eval_balanced_accuracy": 0.7282608695652174,
-      "eval_loss": 0.5141614079475403,
-      "eval_runtime": 146.7922,
-      "eval_samples_per_second": 1.063,
-      "eval_steps_per_second": 0.068,
+      "eval_accuracy": 0.6217948717948718,
+      "eval_balanced_accuracy": 0.6318867924528302,
+      "eval_loss": 0.6179038882255554,
+      "eval_runtime": 111.4097,
+      "eval_samples_per_second": 1.4,
+      "eval_steps_per_second": 0.09,
       "step": 828
     },
     {
       "epoch": 19.0,
-      "eval_accuracy": 0.7115384615384616,
-      "eval_balanced_accuracy": 0.7167124227865478,
-      "eval_loss": 0.5163176655769348,
-      "eval_runtime": 112.8422,
-      "eval_samples_per_second": 1.382,
-      "eval_steps_per_second": 0.089,
+      "eval_accuracy": 0.6282051282051282,
+      "eval_balanced_accuracy": 0.63264221158958,
+      "eval_loss": 0.6252104640007019,
+      "eval_runtime": 111.3148,
+      "eval_samples_per_second": 1.401,
+      "eval_steps_per_second": 0.09,
       "step": 874
     },
     {
       "epoch": 20.0,
-      "eval_accuracy": 0.7115384615384616,
-      "eval_balanced_accuracy": 0.7113095238095238,
-      "eval_loss": 0.5302315950393677,
-      "eval_runtime": 112.7072,
-      "eval_samples_per_second": 1.384,
-      "eval_steps_per_second": 0.089,
+      "eval_accuracy": 0.6282051282051282,
+      "eval_balanced_accuracy": 0.63264221158958,
+      "eval_loss": 0.6249805688858032,
+      "eval_runtime": 111.384,
+      "eval_samples_per_second": 1.401,
+      "eval_steps_per_second": 0.09,
       "step": 920
     },
     {
       "epoch": 20.0,
       "step": 920,
-      "total_flos": 4.976764349571072e+17,
-      "train_loss": 0.4557167053222656,
-      "train_runtime": 45521.9577,
-      "train_samples_per_second": 0.321,
-      "train_steps_per_second": 0.02
+      "total_flos": 4.970048339828736e+17,
+      "train_loss": 0.6203255363132643,
+      "train_runtime": 33373.6734,
+      "train_samples_per_second": 0.437,
+      "train_steps_per_second": 0.028
     }
   ],
   "logging_steps": 500,
@@ -233,7 +233,7 @@
   "stateful_callbacks": {
     "EarlyStoppingCallback": {
       "args": {
-        "early_stopping_patience": 10,
+        "early_stopping_patience": 19,
         "early_stopping_threshold": 0.0
       },
       "attributes": {
@@ -251,7 +251,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.976764349571072e+17,
+  "total_flos": 4.970048339828736e+17,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null