Model save

README.md

@@ -2,7 +2,6 @@
 license: apache-2.0
 library_name: peft
 tags:
-- axolotl
 - generated_from_trainer
 base_model: mistralai/Mixtral-8x7B-Instruct-v0.1
 model-index:
@@ -25,13 +24,13 @@ chat_template: inst
 dataset_prepared_path: last_run_prepared
 datasets:
 - conversation: mistral
-  path: 4d024e8e9a7d4c788853a531226680a8/./data/with_function_response/original_clean/function_used_training.jsonl
+  path: 722c5ac4af9b4ac09cdd250a281284f5/./data/with_function_response/original_clean/function_used_training.jsonl
   type: sharegpt
 - conversation: mistral
-  path: 4d024e8e9a7d4c788853a531226680a8/./data/with_function_response/original_clean/function_not_used_training.jsonl
+  path: 722c5ac4af9b4ac09cdd250a281284f5/./data/with_function_response/original_clean/function_not_used_training.jsonl
   type: sharegpt
 - conversation: mistral
-  path: 4d024e8e9a7d4c788853a531226680a8/./data/with_function_response/parallel_call/parallel_data_training.jsonl
+  path: 722c5ac4af9b4ac09cdd250a281284f5/./data/with_function_response/parallel_call/parallel_data_training.jsonl
   type: sharegpt
 debug: null
 eval_max_new_tokens: 256
@@ -52,6 +51,9 @@ logging_steps: 1
 lora_alpha: 64
 lora_dropout: 0.05
 lora_model_dir: null
+lora_modules_to_save:
+- embed_tokens
+- lm_head
 lora_r: 32
 lora_target_linear: true
 loss_watchdog_patience: 3
@@ -61,22 +63,22 @@ micro_batch_size: 2
 model_config:
   output_router_logits: true
 model_type: AutoModelForCausalLM
-num_epochs: 4
+num_epochs: 1
 optimizer: paged_adamw_8bit
-output_dir: 4d024e8e9a7d4c788853a531226680a8/model
+output_dir: 722c5ac4af9b4ac09cdd250a281284f5/model
 pad_to_sequence_len: true
 sample_packing: true
 save_steps: 0.1
 sequence_len: 4096
-special_tokens:
-  conversation: '[c]'
-  function: '[f]'
 strict: false
 tf32: false
 tokenizer_type: AutoTokenizer
+tokens:
+- '[f]'
+- '[c]'
 train_on_inputs: false
 trust_remote_code: true
-val_set_size: 0.025
+val_set_size: 0.05
 wandb_log_model: end
 wandb_name: more-tools
 wandb_project: function-call
@@ -89,9 +91,7 @@ weight_decay: 0.0
 
 # special-token-all-linear
 
-This model is a fine-tuned version of [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 0.0801
+This model is a fine-tuned version of [mistralai/Mixtral-8x7B-Instruct-v0.1](https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1) on an unknown dataset.
 
 ## Model description
 
@@ -122,18 +122,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- num_epochs: 4
-
-### Training results
-
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| 2.1829        | 0.01  | 1    | 2.1038          |
-| 0.091         | 0.8   | 151  | 0.0832          |
-| 0.0741        | 1.58  | 302  | 0.0801          |
-| 0.0687        | 2.36  | 453  | 0.0801          |
-| 0.0654        | 3.14  | 604  | 0.0801          |
-
+- num_epochs: 1
 
 ### Framework versions
 

adapter_config.json

@@ -13,20 +13,23 @@
   "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
-  "modules_to_save": null,
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
   "peft_type": "LORA",
   "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "w1",
-    "o_proj",
-    "q_proj",
+    "w2",
     "k_proj",
+    "q_proj",
+    "w1",
     "v_proj",
+    "gate",
     "w3",
-    "w2",
-    "gate"
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ddc484b9de71050d31c0251f3f04ec4c219f9cf7c78f2be25946331265237faa
-size 1938077512
+oid sha256:752f1017a27bd38e1d2b3f9d6687df0af638719a10af88e6dc48d059a5d71d14
+size 3511040576

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "[c]": 32001,
+  "[f]": 32000
+}

config.json

@@ -40,5 +40,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.38.2",
   "use_cache": false,
-  "vocab_size": 32000
+  "vocab_size": 32002
 }

tokenizer.json

@@ -29,6 +29,24 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 32000,
+      "content": "[f]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 32001,
+      "content": "[c]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -25,6 +25,22 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "32000": {
+      "content": "[f]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "32001": {
+      "content": "[c]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e2bf1b5b988d4cfe0c4c0460a349291c90deef362876a889aaa6c018d31b193
+oid sha256:3943d56b3a1e58cef6a9201d4b1b47c36798809881f5ae91a6c377c0528808e3
 size 5688