Model save

README.md

@@ -19,7 +19,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.9174
+- Loss: 0.1851
 
 ## Model description
 
@@ -45,14 +45,23 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 1
+- num_epochs: 10
 - mixed_precision_training: Native AMP
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 2.8727        | 1.0   | 4    | 1.9174          |
+| 2.8727        | 1.0   | 4    | 1.9824          |
+| 2.0991        | 2.0   | 8    | 1.3412          |
+| 1.5585        | 3.0   | 12   | 0.7900          |
+| 0.8816        | 4.0   | 16   | 0.5710          |
+| 0.552         | 5.0   | 20   | 0.4502          |
+| 0.552         | 6.0   | 24   | 0.3272          |
+| 0.3661        | 7.0   | 28   | 0.2535          |
+| 0.2903        | 8.0   | 32   | 0.2082          |
+| 0.1619        | 9.0   | 36   | 0.1888          |
+| 0.2003        | 10.0  | 40   | 0.1851          |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
+    "v_proj",
     "q_proj",
-    "k_proj",
-    "down_proj",
     "gate_proj",
+    "o_proj",
+    "down_proj",
     "up_proj",
-    "v_proj"
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4024d95aa023dac0dcb3f335390e8cc2f688c02d9ebf9339aa72b882a723ad5a
+oid sha256:f36b592910ba77ca2a1d94dd88b931fbd002a984d20cc17f7876cadfa7dbbfba
 size 639691872

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 2.2938016653060913,
-    "train_runtime": 14.4923,
+    "epoch": 10.0,
+    "train_loss": 0.7830681726336479,
+    "train_runtime": 86.987,
     "train_samples": 10,
-    "train_samples_per_second": 0.69,
-    "train_steps_per_second": 0.276
+    "train_samples_per_second": 1.15,
+    "train_steps_per_second": 0.46
 }

runs/Apr23_15-25-05_SYS-4029GP-TRT/events.out.tfevents.1713857118.SYS-4029GP-TRT.1442820.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93a89346934b9248a83b48e436da8f28a2555c4918ed941e583ec67380b89137
+size 9981

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "train_loss": 2.2938016653060913,
-    "train_runtime": 14.4923,
+    "epoch": 10.0,
+    "train_loss": 0.7830681726336479,
+    "train_runtime": 86.987,
     "train_samples": 10,
-    "train_samples_per_second": 0.69,
-    "train_steps_per_second": 0.276
+    "train_samples_per_second": 1.15,
+    "train_steps_per_second": 0.46
 }

trainer_state.json

@@ -1,44 +1,172 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 4,
+  "global_step": 40,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.25,
-      "grad_norm": 29891.4296875,
-      "learning_rate": 0.0002,
+      "grad_norm": 30026.80859375,
+      "learning_rate": 5e-05,
       "loss": 2.8727,
       "step": 1
     },
     {
       "epoch": 1.0,
-      "eval_loss": 1.9173675775527954,
-      "eval_runtime": 1.3663,
-      "eval_samples_per_second": 7.319,
-      "eval_steps_per_second": 0.732,
+      "eval_loss": 1.982383131980896,
+      "eval_runtime": 1.3595,
+      "eval_samples_per_second": 7.356,
+      "eval_steps_per_second": 0.736,
       "step": 4
     },
     {
-      "epoch": 1.0,
-      "step": 4,
-      "total_flos": 114055256039424.0,
-      "train_loss": 2.2938016653060913,
-      "train_runtime": 14.4923,
-      "train_samples_per_second": 0.69,
-      "train_steps_per_second": 0.276
+      "epoch": 1.25,
+      "grad_norm": 19052.849609375,
+      "learning_rate": 0.00019961946980917456,
+      "loss": 2.0991,
+      "step": 5
+    },
+    {
+      "epoch": 2.0,
+      "eval_loss": 1.341205358505249,
+      "eval_runtime": 1.3571,
+      "eval_samples_per_second": 7.368,
+      "eval_steps_per_second": 0.737,
+      "step": 8
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 17590.10546875,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 1.5585,
+      "step": 10
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 0.7899824380874634,
+      "eval_runtime": 1.3639,
+      "eval_samples_per_second": 7.332,
+      "eval_steps_per_second": 0.733,
+      "step": 12
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 19440.537109375,
+      "learning_rate": 0.0001573576436351046,
+      "loss": 0.8816,
+      "step": 15
+    },
+    {
+      "epoch": 4.0,
+      "eval_loss": 0.5710309743881226,
+      "eval_runtime": 1.3546,
+      "eval_samples_per_second": 7.382,
+      "eval_steps_per_second": 0.738,
+      "step": 16
+    },
+    {
+      "epoch": 5.0,
+      "grad_norm": 69964.546875,
+      "learning_rate": 0.00011736481776669306,
+      "loss": 0.552,
+      "step": 20
+    },
+    {
+      "epoch": 5.0,
+      "eval_loss": 0.45021525025367737,
+      "eval_runtime": 1.3712,
+      "eval_samples_per_second": 7.293,
+      "eval_steps_per_second": 0.729,
+      "step": 20
+    },
+    {
+      "epoch": 6.0,
+      "eval_loss": 0.32722944021224976,
+      "eval_runtime": 1.3552,
+      "eval_samples_per_second": 7.379,
+      "eval_steps_per_second": 0.738,
+      "step": 24
+    },
+    {
+      "epoch": 6.25,
+      "grad_norm": 23555.736328125,
+      "learning_rate": 7.411809548974792e-05,
+      "loss": 0.3661,
+      "step": 25
+    },
+    {
+      "epoch": 7.0,
+      "eval_loss": 0.253467857837677,
+      "eval_runtime": 1.3644,
+      "eval_samples_per_second": 7.329,
+      "eval_steps_per_second": 0.733,
+      "step": 28
+    },
+    {
+      "epoch": 7.5,
+      "grad_norm": 16426.447265625,
+      "learning_rate": 3.5721239031346066e-05,
+      "loss": 0.2903,
+      "step": 30
+    },
+    {
+      "epoch": 8.0,
+      "eval_loss": 0.20816144347190857,
+      "eval_runtime": 1.3588,
+      "eval_samples_per_second": 7.36,
+      "eval_steps_per_second": 0.736,
+      "step": 32
+    },
+    {
+      "epoch": 8.75,
+      "grad_norm": 15496.693359375,
+      "learning_rate": 9.369221296335006e-06,
+      "loss": 0.1619,
+      "step": 35
+    },
+    {
+      "epoch": 9.0,
+      "eval_loss": 0.18882343173027039,
+      "eval_runtime": 1.3661,
+      "eval_samples_per_second": 7.32,
+      "eval_steps_per_second": 0.732,
+      "step": 36
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 21258.580078125,
+      "learning_rate": 0.0,
+      "loss": 0.2003,
+      "step": 40
+    },
+    {
+      "epoch": 10.0,
+      "eval_loss": 0.1850774735212326,
+      "eval_runtime": 1.3683,
+      "eval_samples_per_second": 7.308,
+      "eval_steps_per_second": 0.731,
+      "step": 40
+    },
+    {
+      "epoch": 10.0,
+      "step": 40,
+      "total_flos": 1230205107683328.0,
+      "train_loss": 0.7830681726336479,
+      "train_runtime": 86.987,
+      "train_samples_per_second": 1.15,
+      "train_steps_per_second": 0.46
     }
   ],
   "logging_steps": 5,
-  "max_steps": 4,
+  "max_steps": 40,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 10,
   "save_steps": 100,
-  "total_flos": 114055256039424.0,
+  "total_flos": 1230205107683328.0,
   "train_batch_size": 3,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d912b6eb6f3676ce73ac43c71402e040dacaf18e914cbeb05b78071b7202e9b
+oid sha256:df369c4695365c83e4e8cb4759b2d2048d42e34b74a587d0dfa24d6df548f120
 size 4984