Jessiecs/llama-2-7b-a3-4

README.md

@@ -50,8 +50,8 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- PEFT 0.8.2
+- PEFT 0.9.1.dev0
 - Transformers 4.39.0.dev0
 - Pytorch 2.1.0+cu121
-- Datasets 2.17.1
+- Datasets 2.18.0
 - Tokenizers 0.15.2

adapter_config.json

@@ -22,5 +22,6 @@
     "base_layer"
   ],
   "task_type": "CAUSAL_LM",
+  "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74fe0d788196c4869eede2cdf2d976288cc7c2ffdfbd37437e0f9f1d4346d64f
+oid sha256:88250486d283fc41c276492b96bda0af9f37ab2a0ae6fceb906439b8aca0b319
 size 479942472

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.2,
-    "total_flos": 3146427009073152.0,
-    "train_loss": 0.5001711931079627,
-    "train_runtime": 93.2825,
-    "train_samples_per_second": 0.858,
-    "train_steps_per_second": 0.214
+    "epoch": 0.62,
+    "total_flos": 1616765879623680.0,
+    "train_loss": 1.9969253599643708,
+    "train_runtime": 79.7523,
+    "train_samples_per_second": 1.003,
+    "train_steps_per_second": 0.251
 }

runs/Mar04_09-17-43_806464d2401f/events.out.tfevents.1709543863.806464d2401f.1315.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f07783ebaefbeca8348320972d0fc63020fba7c9b90e199d55f06ca315593c1f
+size 9582

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 3.2,
-    "total_flos": 3146427009073152.0,
-    "train_loss": 0.5001711931079627,
-    "train_runtime": 93.2825,
-    "train_samples_per_second": 0.858,
-    "train_steps_per_second": 0.214
+    "epoch": 0.62,
+    "total_flos": 1616765879623680.0,
+    "train_loss": 1.9969253599643708,
+    "train_runtime": 79.7523,
+    "train_samples_per_second": 1.003,
+    "train_steps_per_second": 0.251
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.2,
+  "epoch": 0.625,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -9,161 +9,161 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.16,
-      "grad_norm": 0.8734284043312073,
+      "epoch": 0.03,
+      "grad_norm": 2.219240188598633,
       "learning_rate": 0.0001,
-      "loss": 1.5075,
+      "loss": 2.4598,
       "step": 1
     },
     {
-      "epoch": 0.32,
-      "grad_norm": 0.8822978138923645,
+      "epoch": 0.06,
+      "grad_norm": 2.17581844329834,
       "learning_rate": 0.0002,
-      "loss": 1.7499,
+      "loss": 2.7545,
       "step": 2
     },
     {
-      "epoch": 0.48,
-      "grad_norm": 0.7636933922767639,
+      "epoch": 0.09,
+      "grad_norm": 1.500047206878662,
       "learning_rate": 0.00018888888888888888,
-      "loss": 1.4175,
+      "loss": 2.2534,
       "step": 3
     },
     {
-      "epoch": 0.64,
-      "grad_norm": 0.8630443811416626,
+      "epoch": 0.12,
+      "grad_norm": 1.3228497505187988,
       "learning_rate": 0.00017777777777777779,
-      "loss": 1.0686,
+      "loss": 2.1453,
       "step": 4
     },
     {
-      "epoch": 0.8,
-      "grad_norm": 0.9574275612831116,
+      "epoch": 0.16,
+      "grad_norm": 1.1492576599121094,
       "learning_rate": 0.0001666666666666667,
-      "loss": 0.7723,
+      "loss": 1.9039,
       "step": 5
     },
     {
-      "epoch": 0.96,
-      "grad_norm": 1.6739799976348877,
+      "epoch": 0.19,
+      "grad_norm": 1.4337806701660156,
       "learning_rate": 0.00015555555555555556,
-      "loss": 0.5065,
+      "loss": 2.0063,
       "step": 6
     },
     {
-      "epoch": 1.12,
-      "grad_norm": 1.1822175979614258,
+      "epoch": 0.22,
+      "grad_norm": 0.8797003030776978,
       "learning_rate": 0.00014444444444444444,
-      "loss": 0.4154,
+      "loss": 1.6465,
       "step": 7
     },
     {
-      "epoch": 1.28,
-      "grad_norm": 0.6320245862007141,
+      "epoch": 0.25,
+      "grad_norm": 0.8188653588294983,
       "learning_rate": 0.00013333333333333334,
-      "loss": 0.2728,
+      "loss": 2.0039,
       "step": 8
     },
     {
-      "epoch": 1.44,
-      "grad_norm": 0.6290435194969177,
+      "epoch": 0.28,
+      "grad_norm": 1.0144400596618652,
       "learning_rate": 0.00012222222222222224,
-      "loss": 0.2711,
+      "loss": 2.285,
       "step": 9
     },
     {
-      "epoch": 1.6,
-      "grad_norm": 0.3685467541217804,
+      "epoch": 0.31,
+      "grad_norm": 1.009358286857605,
       "learning_rate": 0.00011111111111111112,
-      "loss": 0.1759,
+      "loss": 2.0178,
       "step": 10
     },
     {
-      "epoch": 1.76,
-      "grad_norm": 0.4463210105895996,
+      "epoch": 0.34,
+      "grad_norm": 1.2635082006454468,
       "learning_rate": 0.0001,
-      "loss": 0.2825,
+      "loss": 1.8403,
       "step": 11
     },
     {
-      "epoch": 1.92,
-      "grad_norm": 0.4311699867248535,
+      "epoch": 0.38,
+      "grad_norm": 1.0166571140289307,
       "learning_rate": 8.888888888888889e-05,
-      "loss": 0.2496,
+      "loss": 1.9239,
       "step": 12
     },
     {
-      "epoch": 2.08,
-      "grad_norm": 0.3269866406917572,
+      "epoch": 0.41,
+      "grad_norm": 0.8766961097717285,
       "learning_rate": 7.777777777777778e-05,
-      "loss": 0.1629,
+      "loss": 1.9722,
       "step": 13
     },
     {
-      "epoch": 2.24,
-      "grad_norm": 0.25907543301582336,
+      "epoch": 0.44,
+      "grad_norm": 0.8238506317138672,
       "learning_rate": 6.666666666666667e-05,
-      "loss": 0.1562,
+      "loss": 2.0839,
       "step": 14
     },
     {
-      "epoch": 2.4,
-      "grad_norm": 0.2790839672088623,
+      "epoch": 0.47,
+      "grad_norm": 0.7714257836341858,
       "learning_rate": 5.555555555555556e-05,
-      "loss": 0.2234,
+      "loss": 1.6972,
       "step": 15
     },
     {
-      "epoch": 2.56,
-      "grad_norm": 0.29331278800964355,
+      "epoch": 0.5,
+      "grad_norm": 0.8575778007507324,
       "learning_rate": 4.4444444444444447e-05,
-      "loss": 0.1783,
+      "loss": 1.8562,
       "step": 16
     },
     {
-      "epoch": 2.72,
-      "grad_norm": 0.26256364583969116,
+      "epoch": 0.53,
+      "grad_norm": 0.8333547115325928,
       "learning_rate": 3.3333333333333335e-05,
-      "loss": 0.1776,
+      "loss": 1.8273,
       "step": 17
     },
     {
-      "epoch": 2.88,
-      "grad_norm": 0.2941267490386963,
+      "epoch": 0.56,
+      "grad_norm": 0.7584834098815918,
       "learning_rate": 2.2222222222222223e-05,
-      "loss": 0.1771,
+      "loss": 1.6062,
       "step": 18
     },
     {
-      "epoch": 3.04,
-      "grad_norm": 0.269639790058136,
+      "epoch": 0.59,
+      "grad_norm": 0.7320359349250793,
       "learning_rate": 1.1111111111111112e-05,
-      "loss": 0.136,
+      "loss": 1.6753,
       "step": 19
     },
     {
-      "epoch": 3.2,
-      "grad_norm": 0.36100998520851135,
+      "epoch": 0.62,
+      "grad_norm": 0.835627555847168,
       "learning_rate": 0.0,
-      "loss": 0.1024,
+      "loss": 1.9796,
       "step": 20
     },
     {
-      "epoch": 3.2,
+      "epoch": 0.62,
       "step": 20,
-      "total_flos": 3146427009073152.0,
-      "train_loss": 0.5001711931079627,
-      "train_runtime": 93.2825,
-      "train_samples_per_second": 0.858,
-      "train_steps_per_second": 0.214
+      "total_flos": 1616765879623680.0,
+      "train_loss": 1.9969253599643708,
+      "train_runtime": 79.7523,
+      "train_samples_per_second": 1.003,
+      "train_steps_per_second": 0.251
     }
   ],
   "logging_steps": 1,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 4,
+  "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 3146427009073152.0,
+  "total_flos": 1616765879623680.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c80c831d7943e08a8eaaafc45a3783d9c4cb7bd0911dfb150712957d123b1631
+oid sha256:9d31c9963efb464f6b6e3ae1af32a5b56c57ec84938354490ffdd2333d234b7f
 size 4920