sujithatz/finbot-transofrmer-based-phi3.5_adapter

README.md

@@ -18,7 +18,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [microsoft/Phi-3.5-mini-instruct](https://huggingface.co/microsoft/Phi-3.5-mini-instruct) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7215
+- Loss: 0.3840
 
 ## Model description
 
@@ -38,11 +38,11 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0002
-- train_batch_size: 8
-- eval_batch_size: 8
+- train_batch_size: 4
+- eval_batch_size: 4
 - seed: 3407
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 32
+- total_train_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 5
@@ -52,30 +52,30 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch   | Step | Validation Loss |
 |:-------------:|:-------:|:----:|:---------------:|
-| 1.4909        | 1.1765  | 5    | 1.3154          |
-| 0.9704        | 2.3529  | 10   | 0.9087          |
-| 0.6673        | 3.5294  | 15   | 0.6343          |
-| 0.4418        | 4.7059  | 20   | 0.5075          |
-| 0.3375        | 5.8824  | 25   | 0.4491          |
-| 0.3033        | 7.0588  | 30   | 0.4069          |
-| 0.244         | 8.2353  | 35   | 0.3828          |
-| 0.2285        | 9.4118  | 40   | 0.3759          |
-| 0.1519        | 10.5882 | 45   | 0.3896          |
-| 0.1334        | 11.7647 | 50   | 0.4114          |
-| 0.099         | 12.9412 | 55   | 0.4291          |
-| 0.0823        | 14.1176 | 60   | 0.4610          |
-| 0.06          | 15.2941 | 65   | 0.4894          |
-| 0.0548        | 16.4706 | 70   | 0.5345          |
-| 0.0437        | 17.6471 | 75   | 0.5747          |
-| 0.0409        | 18.8235 | 80   | 0.6059          |
-| 0.0386        | 20.0    | 85   | 0.6349          |
-| 0.0272        | 21.1765 | 90   | 0.6590          |
-| 0.0262        | 22.3529 | 95   | 0.6933          |
-| 0.0303        | 23.5294 | 100  | 0.6960          |
-| 0.0249        | 24.7059 | 105  | 0.7021          |
-| 0.0291        | 25.8824 | 110  | 0.7173          |
-| 0.0255        | 27.0588 | 115  | 0.7195          |
-| 0.0208        | 28.2353 | 120  | 0.7215          |
+| 1.5071        | 0.5882  | 5    | 1.4674          |
+| 1.1659        | 1.1765  | 10   | 1.0849          |
+| 0.894         | 1.7647  | 15   | 0.8655          |
+| 0.7243        | 2.3529  | 20   | 0.6989          |
+| 0.5752        | 2.9412  | 25   | 0.5856          |
+| 0.5724        | 3.5294  | 30   | 0.5257          |
+| 0.4834        | 4.1176  | 35   | 0.4875          |
+| 0.3861        | 4.7059  | 40   | 0.4588          |
+| 0.35          | 5.2941  | 45   | 0.4368          |
+| 0.3126        | 5.8824  | 50   | 0.4251          |
+| 0.367         | 6.4706  | 55   | 0.4080          |
+| 0.2792        | 7.0588  | 60   | 0.3955          |
+| 0.3952        | 7.6471  | 65   | 0.3914          |
+| 0.2854        | 8.2353  | 70   | 0.3784          |
+| 0.3224        | 8.8235  | 75   | 0.3867          |
+| 0.3187        | 9.4118  | 80   | 0.3765          |
+| 0.1675        | 10.0    | 85   | 0.3799          |
+| 0.1888        | 10.5882 | 90   | 0.3858          |
+| 0.2021        | 11.1765 | 95   | 0.3759          |
+| 0.1518        | 11.7647 | 100  | 0.3868          |
+| 0.2075        | 12.3529 | 105  | 0.3915          |
+| 0.1497        | 12.9412 | 110  | 0.3814          |
+| 0.1797        | 13.5294 | 115  | 0.3821          |
+| 0.1606        | 14.1176 | 120  | 0.3840          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,10 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
+    "up_proj",
     "o_proj",
     "down_proj",
-    "gate_up_proj",
-    "qkv_proj"
+    "k_proj",
+    "q_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d0d087ec02232d98e64e6f6b528eebfa7ca7a0bf61f2f00fe1c0991fe80fee6
-size 100697728
+oid sha256:5db02aecfe4e8679acdaf54528b4994462fd68c005d87e3419662df44736163f
+size 35668592

tokenizer.json

@@ -155,12 +155,6 @@
           "id": "A",
           "type_id": 0
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 0
-        }
       }
     ],
     "pair": [
@@ -170,36 +164,14 @@
           "type_id": 0
         }
       },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 0
-        }
-      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
-      },
-      {
-        "SpecialToken": {
-          "id": "<|endoftext|>",
-          "type_id": 1
-        }
       }
     ],
-    "special_tokens": {
-      "<|endoftext|>": {
-        "id": "<|endoftext|>",
-        "ids": [
-          32000
-        ],
-        "tokens": [
-          "<|endoftext|>"
-        ]
-      }
-    }
+    "special_tokens": {}
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -1,6 +1,6 @@
 {
   "add_bos_token": false,
-  "add_eos_token": true,
+  "add_eos_token": false,
   "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
@@ -121,9 +121,9 @@
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,
-  "model_max_length": 131072,
+  "model_max_length": 300,
   "pad_token": "<unk>",
-  "padding_side": "left",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae64cfbc9fafa79992f8f1dbc59d731406c1a3c9322aa24ad9ba448b90f16c6e
+oid sha256:98e6d562609b6199ed2dc3c6d856a06eb939e0a4a026ced7b500b84e4f340b13
 size 5432