End of training

README.md

@@ -2,6 +2,7 @@
 license: apache-2.0
 library_name: peft
 tags:
+- text-generation
 - alignment-handbook
 - generated_from_trainer
 - trl

adapter_config.json

@@ -20,11 +20,11 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
+    "v_proj",
+    "k_proj",
     "o_proj",
     "q_proj",
-    "v_proj",
-    "k_proj"
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:68fa615f2e2431c2a4128c632f3e5b71376ad7a9a3fbb213177c8693f0e6db6d
+oid sha256:bb6d6addf9a5964eb8913cf4d298675cb9044b190b16c9f90bc3a3143890a304
 size 616639296

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 5.9171597633136095,
+    "epoch": 5.923076923076923,
     "eval_loss": 1.1831614971160889,
-    "eval_runtime": 5.5457,
+    "eval_runtime": 5.4341,
     "eval_samples": 169,
-    "eval_samples_per_second": 16.049,
-    "eval_steps_per_second": 2.164,
-    "total_flos": 1.75885655212032e+17,
-    "train_loss": 0.2793775268793106,
-    "train_runtime": 1218.5958,
+    "eval_samples_per_second": 16.378,
+    "eval_steps_per_second": 2.208,
+    "total_flos": 1.7606154086724403e+17,
+    "train_loss": 4.579017792905604e-05,
+    "train_runtime": 1.6348,
     "train_samples": 1346,
-    "train_samples_per_second": 3.282,
-    "train_steps_per_second": 0.821
+    "train_samples_per_second": 2446.747,
+    "train_steps_per_second": 611.687
 }

config.json

@@ -4,8 +4,8 @@
     "MistralForCausalLM"
   ],
   "attention_dropout": 0.0,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
+  "bos_token_id": 32000,
+  "eos_token_id": 32001,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.02,
@@ -15,11 +15,12 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
+  "pad_token_id": 32001,
   "rms_norm_eps": 1e-05,
   "rope_theta": 1000000.0,
   "sliding_window": null,
   "tie_word_embeddings": false,
-  "torch_dtype": "float32",
+  "torch_dtype": "float16",
   "transformers_version": "4.40.0",
   "use_cache": true,
   "vocab_size": 32002

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 5.9171597633136095,
+    "epoch": 5.923076923076923,
     "eval_loss": 1.1831614971160889,
-    "eval_runtime": 5.5457,
+    "eval_runtime": 5.4341,
     "eval_samples": 169,
-    "eval_samples_per_second": 16.049,
-    "eval_steps_per_second": 2.164
+    "eval_samples_per_second": 16.378,
+    "eval_steps_per_second": 2.208
 }

special_tokens_map.json

@@ -1,19 +1,23 @@
 {
-  "bos_token": {
-    "content": "<s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "</s>",
+  "additional_special_tokens": [
+    {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|im_start|>",
+  "eos_token": "<|im_end|>",
+  "pad_token": "<|im_end|>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -32,21 +32,21 @@
     },
     {
       "id": 32000,
-      "content": "<new_token1>",
+      "content": "<|im_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true,
-      "special": false
+      "normalized": false,
+      "special": true
     },
     {
       "id": 32001,
-      "content": "<new_token2>",
+      "content": "<|im_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true,
-      "special": false
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -27,30 +27,33 @@
       "special": true
     },
     "32000": {
-      "content": "<new_token1>",
+      "content": "<|im_start|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "32001": {
-      "content": "<new_token2>",
+      "content": "<|im_end|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     }
   },
-  "additional_special_tokens": [],
-  "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "</s>",
+  "eos_token": "<|im_end|>",
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "</s>",
+  "pad_token": "<|im_end|>",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 5.9171597633136095,
-    "total_flos": 1.75885655212032e+17,
-    "train_loss": 0.2793775268793106,
-    "train_runtime": 1218.5958,
+    "epoch": 5.923076923076923,
+    "total_flos": 1.7606154086724403e+17,
+    "train_loss": 4.579017792905604e-05,
+    "train_runtime": 1.6348,
     "train_samples": 1346,
-    "train_samples_per_second": 3.282,
-    "train_steps_per_second": 0.821
+    "train_samples_per_second": 2446.747,
+    "train_steps_per_second": 611.687
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.9171597633136095,
+  "epoch": 5.923076923076923,
   "eval_steps": 25,
-  "global_step": 1000,
+  "global_step": 1001,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -609,13 +609,13 @@
       "step": 1000
     },
     {
-      "epoch": 5.9171597633136095,
-      "step": 1000,
-      "total_flos": 1.75885655212032e+17,
-      "train_loss": 0.2793775268793106,
-      "train_runtime": 1218.5958,
-      "train_samples_per_second": 3.282,
-      "train_steps_per_second": 0.821
+      "epoch": 5.923076923076923,
+      "step": 1001,
+      "total_flos": 1.7606154086724403e+17,
+      "train_loss": 4.579017792905604e-05,
+      "train_runtime": 1.6348,
+      "train_samples_per_second": 2446.747,
+      "train_steps_per_second": 611.687
     }
   ],
   "logging_steps": 25,
@@ -623,7 +623,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 6,
   "save_steps": 25,
-  "total_flos": 1.75885655212032e+17,
+  "total_flos": 1.7606154086724403e+17,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null