Upload folder using huggingface_hub

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: unsloth/mistral-7b-instruct-v0.3-bnb-4bit
+base_model: mistralai/Mistral-7B-Instruct-v0.3
 library_name: peft
 ---
 
@@ -199,4 +199,4 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
-- PEFT 0.15.2
+- PEFT 0.15.0

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "mistralai/mistral-7b-instruct-v0.3",
+  "base_model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -10,23 +10,26 @@
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
-  "lora_alpha": 256,
-  "lora_dropout": 0.2,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 128,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "k_proj",
-    "gate_proj",
+    "lm_head",
     "up_proj",
+    "v_proj",
+    "k_proj",
     "down_proj",
+    "fc1",
+    "fc2",
     "o_proj",
-    "v_proj"
+    "gate_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5688e3fecd65ac31fbe0f744b7374da6823663ec32ea1fddfe7fdbb2ff07eb61
-size 671150064
+oid sha256:96345905565c6c5c711ee6ad708b7466859bd150df9d1895668af8f39e0be019
+size 438627360

special_tokens_map.json

@@ -13,13 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "[control_768]",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
+  "pad_token": "</s>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

trainer_state.json

@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 13.132231404958677,
+  "eval_steps": 500,
+  "global_step": 400,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.3305785123966942,
+      "grad_norm": 2.2538764476776123,
+      "learning_rate": 0.00027,
+      "loss": 1.9421,
+      "step": 10
+    },
+    {
+      "epoch": 0.6611570247933884,
+      "grad_norm": 2.071784019470215,
+      "learning_rate": 0.00029834218059022024,
+      "loss": 1.7991,
+      "step": 20
+    },
+    {
+      "epoch": 0.9917355371900827,
+      "grad_norm": 2.1005189418792725,
+      "learning_rate": 0.00029265847744427303,
+      "loss": 1.7063,
+      "step": 30
+    },
+    {
+      "epoch": 1.2975206611570247,
+      "grad_norm": 2.115266799926758,
+      "learning_rate": 0.00028308342291654174,
+      "loss": 1.1965,
+      "step": 40
+    },
+    {
+      "epoch": 1.6280991735537191,
+      "grad_norm": 2.5068259239196777,
+      "learning_rate": 0.00026987819953423867,
+      "loss": 1.0111,
+      "step": 50
+    },
+    {
+      "epoch": 1.9586776859504131,
+      "grad_norm": 2.9106173515319824,
+      "learning_rate": 0.00025340301136778483,
+      "loss": 1.0334,
+      "step": 60
+    },
+    {
+      "epoch": 2.2644628099173554,
+      "grad_norm": 2.115727424621582,
+      "learning_rate": 0.0002341072586027509,
+      "loss": 0.6771,
+      "step": 70
+    },
+    {
+      "epoch": 2.5950413223140494,
+      "grad_norm": 2.2171154022216797,
+      "learning_rate": 0.00021251727907429355,
+      "loss": 0.4238,
+      "step": 80
+    },
+    {
+      "epoch": 2.925619834710744,
+      "grad_norm": 1.6520980596542358,
+      "learning_rate": 0.00018922199114307294,
+      "loss": 0.4043,
+      "step": 90
+    },
+    {
+      "epoch": 3.231404958677686,
+      "grad_norm": 2.3576242923736572,
+      "learning_rate": 0.00016485682953756942,
+      "loss": 0.2847,
+      "step": 100
+    },
+    {
+      "epoch": 3.56198347107438,
+      "grad_norm": 1.8273522853851318,
+      "learning_rate": 0.00014008641235114508,
+      "loss": 0.2007,
+      "step": 110
+    },
+    {
+      "epoch": 3.8925619834710745,
+      "grad_norm": 1.627296805381775,
+      "learning_rate": 0.00011558641199298727,
+      "loss": 0.138,
+      "step": 120
+    },
+    {
+      "epoch": 4.198347107438017,
+      "grad_norm": 1.7023773193359375,
+      "learning_rate": 9.202512460613219e-05,
+      "loss": 0.151,
+      "step": 130
+    },
+    {
+      "epoch": 4.528925619834711,
+      "grad_norm": 1.108082890510559,
+      "learning_rate": 7.004524069080096e-05,
+      "loss": 0.0575,
+      "step": 140
+    },
+    {
+      "epoch": 4.859504132231405,
+      "grad_norm": 0.7116100192070007,
+      "learning_rate": 5.024631418292274e-05,
+      "loss": 0.0528,
+      "step": 150
+    },
+    {
+      "epoch": 5.1652892561983474,
+      "grad_norm": 0.428362637758255,
+      "learning_rate": 3.316840818568315e-05,
+      "loss": 0.0432,
+      "step": 160
+    },
+    {
+      "epoch": 5.4958677685950414,
+      "grad_norm": 0.38093993067741394,
+      "learning_rate": 1.927736345590839e-05,
+      "loss": 0.0171,
+      "step": 170
+    },
+    {
+      "epoch": 5.8264462809917354,
+      "grad_norm": 0.5677882432937622,
+      "learning_rate": 8.952091482575824e-06,
+      "loss": 0.0179,
+      "step": 180
+    },
+    {
+      "epoch": 6.132231404958677,
+      "grad_norm": 0.27657485008239746,
+      "learning_rate": 2.4742387691426445e-06,
+      "loss": 0.0117,
+      "step": 190
+    },
+    {
+      "epoch": 6.462809917355372,
+      "grad_norm": 0.14065445959568024,
+      "learning_rate": 2.0504251129649374e-08,
+      "loss": 0.0109,
+      "step": 200
+    },
+    {
+      "epoch": 6.991735537190083,
+      "grad_norm": 0.25906819105148315,
+      "learning_rate": 0.00015124008823666873,
+      "loss": 0.0089,
+      "step": 210
+    },
+    {
+      "epoch": 7.330578512396694,
+      "grad_norm": 0.5590444207191467,
+      "learning_rate": 0.00013884937419991687,
+      "loss": 0.0148,
+      "step": 220
+    },
+    {
+      "epoch": 7.661157024793388,
+      "grad_norm": 0.36452558636665344,
+      "learning_rate": 0.00012653483024396533,
+      "loss": 0.0145,
+      "step": 230
+    },
+    {
+      "epoch": 7.991735537190083,
+      "grad_norm": 1.2531826496124268,
+      "learning_rate": 0.00011438057719081671,
+      "loss": 0.0174,
+      "step": 240
+    },
+    {
+      "epoch": 8.297520661157025,
+      "grad_norm": 0.6274934411048889,
+      "learning_rate": 0.00010246964091307433,
+      "loss": 0.0165,
+      "step": 250
+    },
+    {
+      "epoch": 8.62809917355372,
+      "grad_norm": 0.42108115553855896,
+      "learning_rate": 9.088338518304578e-05,
+      "loss": 0.0172,
+      "step": 260
+    },
+    {
+      "epoch": 8.958677685950413,
+      "grad_norm": 0.641461193561554,
+      "learning_rate": 7.970095587567578e-05,
+      "loss": 0.021,
+      "step": 270
+    },
+    {
+      "epoch": 9.264462809917354,
+      "grad_norm": 0.9046468138694763,
+      "learning_rate": 6.899874032196796e-05,
+      "loss": 0.0118,
+      "step": 280
+    },
+    {
+      "epoch": 9.59504132231405,
+      "grad_norm": 0.4416114091873169,
+      "learning_rate": 5.884984550605782e-05,
+      "loss": 0.0118,
+      "step": 290
+    },
+    {
+      "epoch": 9.925619834710744,
+      "grad_norm": 1.0179201364517212,
+      "learning_rate": 4.9323598670382096e-05,
+      "loss": 0.0123,
+      "step": 300
+    },
+    {
+      "epoch": 10.231404958677686,
+      "grad_norm": 0.34066879749298096,
+      "learning_rate": 4.0485073740315574e-05,
+      "loss": 0.0065,
+      "step": 310
+    },
+    {
+      "epoch": 10.561983471074381,
+      "grad_norm": 0.44902798533439636,
+      "learning_rate": 3.239464680327706e-05,
+      "loss": 0.0063,
+      "step": 320
+    },
+    {
+      "epoch": 10.892561983471074,
+      "grad_norm": 0.10280103981494904,
+      "learning_rate": 2.5107583678831445e-05,
+      "loss": 0.0058,
+      "step": 330
+    },
+    {
+      "epoch": 11.198347107438016,
+      "grad_norm": 0.036108873784542084,
+      "learning_rate": 1.8673662397103577e-05,
+      "loss": 0.0056,
+      "step": 340
+    },
+    {
+      "epoch": 11.52892561983471,
+      "grad_norm": 0.048432257026433945,
+      "learning_rate": 1.3136833164357929e-05,
+      "loss": 0.0022,
+      "step": 350
+    },
+    {
+      "epoch": 11.859504132231406,
+      "grad_norm": 0.04557422176003456,
+      "learning_rate": 8.53491813852521e-06,
+      "loss": 0.0025,
+      "step": 360
+    },
+    {
+      "epoch": 12.165289256198347,
+      "grad_norm": 0.03517599403858185,
+      "learning_rate": 4.899353065512262e-06,
+      "loss": 0.0018,
+      "step": 370
+    },
+    {
+      "epoch": 12.49586776859504,
+      "grad_norm": 0.07305194437503815,
+      "learning_rate": 2.2549725411822483e-06,
+      "loss": 0.002,
+      "step": 380
+    },
+    {
+      "epoch": 12.826446280991735,
+      "grad_norm": 0.03562126308679581,
+      "learning_rate": 6.198403658829232e-07,
+      "loss": 0.0021,
+      "step": 390
+    },
+    {
+      "epoch": 13.132231404958677,
+      "grad_norm": 0.046182479709386826,
+      "learning_rate": 5.1261503738131425e-09,
+      "loss": 0.0022,
+      "step": 400
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 14,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.281442086756352e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}