Upload folder using huggingface_hub

README.md

@@ -201,4 +201,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 
 ### Framework versions
 
-- PEFT 0.7.1
+- PEFT 0.8.2.dev0

adapter_config.json

@@ -9,19 +9,24 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 32,
+  "lora_alpha": 16,
   "lora_dropout": 0.1,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
     "gate_proj",
-    "up_proj"
+    "v_proj",
+    "o_proj",
+    "down_proj"
   ],
-  "task_type": "CAUSAL_LM"
+  "task_type": "CAUSAL_LM",
+  "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7f9646a1d55dbd2bd9a1a6f37c9cddd2839ef8503f6b9d4479f81656184a1e5
-size 32457544
+oid sha256:23ffcd23f939b8ae254feeaaad38f22ba7ce8bf927bc0fe451f5d6a7ef9c8770
+size 201892112

optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33163348f71f15f7ba7ae85d8c5d726c563c74b750e5511011a240533a5dc310
-size 64958970
+oid sha256:5ac376aa71656d60165dfacbc225416d84500bf617fd13f4ec5d98924936b02f
+size 403891130

rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a97947770dcbcbe97b40b65e15a105e262e8b61b42747d97678ff13b07e989a6
+oid sha256:9de786fccc95f6680835ed37f997115fe91c647d93a65160aee56507bea10ac4
 size 14244

scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bf1e902c3114bb5312c2c178dc4d942be3a06cb5931cabbb4ddb7748d0f53ee
+oid sha256:0fc8f1838e22027d1c8795abfafd0327a77e02ab1b7dd271ee107ff8dcf6d8d1
 size 1064

special_tokens_map.json

@@ -13,7 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "</s>",
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -1,6 +1,11 @@
 {
   "version": "1.0",
-  "truncation": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 1024,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
   "padding": null,
   "added_tokens": [
     {

tokenizer_config.json

@@ -32,7 +32,7 @@
   "eos_token": "</s>",
   "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "</s>",
+  "pad_token": "<unk>",
   "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",

trainer_state.json

@@ -1,218 +1,164 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9990680335507922,
+  "epoch": 0.9846153846153847,
   "eval_steps": 500,
-  "global_step": 67,
+  "global_step": 48,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.03,
+      "epoch": 0.04,
       "learning_rate": 2e-05,
-      "loss": 2.1238,
+      "loss": 2.0318,
       "step": 2
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 2e-05,
-      "loss": 1.5915,
+      "epoch": 0.08,
+      "learning_rate": 1.9906859460363307e-05,
+      "loss": 2.2592,
       "step": 4
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 2e-05,
-      "loss": 1.3459,
+      "epoch": 0.12,
+      "learning_rate": 1.9629172873477995e-05,
+      "loss": 2.3613,
       "step": 6
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 2e-05,
-      "loss": 1.3217,
+      "epoch": 0.16,
+      "learning_rate": 1.917211301505453e-05,
+      "loss": 2.3608,
       "step": 8
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 2e-05,
-      "loss": 1.4143,
+      "epoch": 0.21,
+      "learning_rate": 1.8544194045464888e-05,
+      "loss": 2.4207,
       "step": 10
     },
     {
-      "epoch": 0.18,
-      "learning_rate": 2e-05,
-      "loss": 1.4196,
+      "epoch": 0.25,
+      "learning_rate": 1.77571129070442e-05,
+      "loss": 3.2413,
       "step": 12
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 2e-05,
-      "loss": 1.8695,
+      "epoch": 0.29,
+      "learning_rate": 1.6825531432186545e-05,
+      "loss": 1.9998,
       "step": 14
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 2e-05,
-      "loss": 2.1552,
+      "epoch": 0.33,
+      "learning_rate": 1.5766803221148676e-05,
+      "loss": 2.1659,
       "step": 16
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 2e-05,
-      "loss": 2.119,
+      "epoch": 0.37,
+      "learning_rate": 1.4600650377311523e-05,
+      "loss": 2.2631,
       "step": 18
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 2e-05,
-      "loss": 1.3909,
+      "epoch": 0.41,
+      "learning_rate": 1.3348796121709862e-05,
+      "loss": 2.4385,
       "step": 20
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 2e-05,
-      "loss": 1.466,
+      "epoch": 0.45,
+      "learning_rate": 1.2034560130526341e-05,
+      "loss": 2.7298,
       "step": 22
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 2e-05,
-      "loss": 1.4721,
+      "epoch": 0.49,
+      "learning_rate": 1.0682424133646712e-05,
+      "loss": 3.174,
       "step": 24
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 2e-05,
-      "loss": 1.2456,
+      "epoch": 0.53,
+      "learning_rate": 9.317575866353293e-06,
+      "loss": 1.989,
       "step": 26
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 2e-05,
-      "loss": 1.6284,
+      "epoch": 0.57,
+      "learning_rate": 7.965439869473664e-06,
+      "loss": 2.2519,
       "step": 28
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 2e-05,
-      "loss": 1.8445,
+      "epoch": 0.62,
+      "learning_rate": 6.651203878290139e-06,
+      "loss": 2.4393,
       "step": 30
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 2e-05,
-      "loss": 1.9897,
+      "epoch": 0.66,
+      "learning_rate": 5.399349622688479e-06,
+      "loss": 2.472,
       "step": 32
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 2e-05,
-      "loss": 1.752,
+      "epoch": 0.7,
+      "learning_rate": 4.2331967788513295e-06,
+      "loss": 2.4543,
       "step": 34
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 2e-05,
-      "loss": 1.7072,
+      "epoch": 0.74,
+      "learning_rate": 3.174468567813461e-06,
+      "loss": 3.0773,
       "step": 36
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 2e-05,
-      "loss": 1.3132,
+      "epoch": 0.78,
+      "learning_rate": 2.2428870929558012e-06,
+      "loss": 2.1053,
       "step": 38
     },
     {
-      "epoch": 0.6,
-      "learning_rate": 2e-05,
-      "loss": 1.4741,
+      "epoch": 0.82,
+      "learning_rate": 1.4558059545351144e-06,
+      "loss": 2.2047,
       "step": 40
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 2e-05,
-      "loss": 1.2306,
+      "epoch": 0.86,
+      "learning_rate": 8.278869849454718e-07,
+      "loss": 2.4035,
       "step": 42
     },
     {
-      "epoch": 0.66,
-      "learning_rate": 2e-05,
-      "loss": 1.3345,
+      "epoch": 0.9,
+      "learning_rate": 3.708271265220087e-07,
+      "loss": 2.4278,
       "step": 44
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 2e-05,
-      "loss": 1.6567,
+      "epoch": 0.94,
+      "learning_rate": 9.314053963669245e-08,
+      "loss": 2.4579,
       "step": 46
     },
-    {
-      "epoch": 0.72,
-      "learning_rate": 2e-05,
-      "loss": 1.7786,
-      "step": 48
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 2e-05,
-      "loss": 2.0943,
-      "step": 50
-    },
-    {
-      "epoch": 0.78,
-      "learning_rate": 2e-05,
-      "loss": 1.4141,
-      "step": 52
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 2e-05,
-      "loss": 1.3025,
-      "step": 54
-    },
-    {
-      "epoch": 0.84,
-      "learning_rate": 2e-05,
-      "loss": 1.2642,
-      "step": 56
-    },
-    {
-      "epoch": 0.86,
-      "learning_rate": 2e-05,
-      "loss": 1.4098,
-      "step": 58
-    },
-    {
-      "epoch": 0.89,
-      "learning_rate": 2e-05,
-      "loss": 1.4349,
-      "step": 60
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 2e-05,
-      "loss": 1.5567,
-      "step": 62
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 2e-05,
-      "loss": 1.47,
-      "step": 64
-    },
     {
       "epoch": 0.98,
-      "learning_rate": 2e-05,
-      "loss": 1.4701,
-      "step": 66
+      "learning_rate": 0.0,
+      "loss": 3.0715,
+      "step": 48
     }
   ],
   "logging_steps": 2,
-  "max_steps": 67,
+  "max_steps": 48,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
-  "total_flos": 3701730966675456.0,
+  "total_flos": 2590139366277120.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58bb305b49b47459f1e39ace47638ea466580751902f2c21fb66e56da1088f98
+oid sha256:5d999138696f38636ccf189f975635df01e9f01da9d3577cd82e2d3997a15c8c
 size 4728