Training in progress, step 200

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -12,12 +12,13 @@
   "initializer_range": 0.041666666666666664,
   "intermediate_size": 1536,
   "is_llama_config": true,
-  "max_position_embeddings": 512,
+  "max_position_embeddings": 2048,
   "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 9,
   "num_hidden_layers": 30,
   "num_key_value_heads": 3,
+  "pad_token_id": 128001,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_interleaved": false,
@@ -25,7 +26,7 @@
   "rope_theta": 100000,
   "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.52.3",
   "use_cache": false,
   "vocab_size": 128257
 }

generation_config.json

@@ -3,6 +3,7 @@
   "bos_token_id": 128000,
   "do_sample": true,
   "eos_token_id": 128001,
-  "transformers_version": "4.51.3",
+  "pad_token_id": 128001,
+  "transformers_version": "4.52.3",
   "use_cache": false
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:164f60c9e8686d5d0ea40890b1f0da06c4fa9caf0898cd3c25195a4b9697cdbd
+oid sha256:b1628d7bfe431768021e3f011bf158fee282742c451771a9f3865487f389d3ea
 size 360189624

tokenizer/chat_template.jinja

@@ -0,0 +1,5 @@
+{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>
+
+'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>
+
+' }}

tokenizer/special_tokens_map.json

@@ -12,5 +12,12 @@
     "normalized": false,
     "rstrip": false,
     "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
   }
 }

tokenizer/tokenizer_config.json

@@ -2058,5 +2058,6 @@
     "attention_mask"
   ],
   "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|end_of_text|>",
   "tokenizer_class": "PreTrainedTokenizer"
 }

tokenizer_config.json

@@ -2058,7 +2058,6 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "extra_special_tokens": {},

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0a26781bb09c9b2a490aabec85ccac84a6261daec0bc472ff18933744c10ead2
+oid sha256:069232dcc2172b941f7d39215d752b3d5611304603103f6f44aec8135f03ba78
 size 6904