sujithatz/finbot-transofrmer-based-phi3.5_adapter

README.md

@@ -38,12 +38,13 @@ The following hyperparameters were used during training:
 - learning_rate: 0.0002
 - train_batch_size: 8
 - eval_batch_size: 8
-- seed: 42
+- seed: 3407
 - gradient_accumulation_steps: 4
 - total_train_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- training_steps: 40
+- lr_scheduler_warmup_steps: 5
+- training_steps: 5
 
 ### Training results
 

adapter_config.json

@@ -12,7 +12,7 @@
   "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "loftq_config": {},
+  "loftq_config": null,
   "lora_alpha": 32,
   "lora_dropout": 0.05,
   "megatron_config": null,
@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "o_proj",
-    "down_proj",
-    "gate_up_proj",
-    "qkv_proj",
     "v_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj",
     "k_proj",
-    "q_proj"
+    "o_proj",
+    "up_proj"
   ],
   "task_type": null,
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b80404a3a2d36415f7078686ffa4c6b502772e8282234b557cf72ea50b2078b5
-size 100697728
+oid sha256:443965e8be292a2bcd1ef4837df3c4f3a69ab40d05701c88346764362f778a84
+size 35668592

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0a2be6a1a67e267ee48988636890206a6b852bcb9df9d4ce59bc67b3f8b0854
+oid sha256:e75e7e86a3fc1a236b9d82a1f17d4139e53851f26782b1abb4f2640be7a11dea
 size 5432