Added some alignment back, it was getting a little too rude to the user.

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c6fa5b5d2bb98348b75f1392c4d945a9d17911ec473d4abfce435eac0b070e0
+oid sha256:1ed1000d0f30d87ff6cf4d6c4a2156dd6854b5b81f63008ef073603409215a7b
 size 4899129920

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1800f18a443529363c0758fcce5c034fd54ef9c7b173c09f7734b69d068a19d4
+oid sha256:00cd4a2c5c0fd80dc53187fe2953bdd5d8525a827c81407528ab2afb8bd1bcd2
 size 4915916176

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:65789f29e60e6195c0e830901a5c7f55aa82407d3add49e3267dd97d6d014e2e
+oid sha256:de55bc93a548d868ef3c70b990067f87de12d3e90240279f2e4ed909af3ae344
 size 4681034848

special_tokens_map.json

@@ -13,6 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
+  "pad_token": {
+    "content": "[control_768]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.json

@@ -95,7 +95,7 @@
     },
     {
       "id": 10,
-      "content": "[IMG]",
+      "content": "[control_8]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 11,
-      "content": "[PREFIX]",
+      "content": "[control_9]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 12,
-      "content": "[MIDDLE]",
+      "content": "[control_10]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 13,
-      "content": "[SUFFIX]",
+      "content": "[control_11]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,7 +131,7 @@
     },
     {
       "id": 14,
-      "content": "[IMG_BREAK]",
+      "content": "[control_12]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -140,7 +140,7 @@
     },
     {
       "id": 15,
-      "content": "[IMG_END]",
+      "content": "[control_13]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -6764,188 +6764,192 @@
     },
     {
       "id": 751,
-      "content": "[REFERENCE_DOC_19]",
+      "content": "[control_749]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 752,
-      "content": "[REFERENCE_DOC_18]",
+      "content": "[control_750]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 753,
-      "content": "[REFERENCE_DOC_17]",
+      "content": "[control_751]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 754,
-      "content": "[REFERENCE_DOC_16]",
+      "content": "[control_752]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 755,
-      "content": "[REFERENCE_DOC_15]",
+      "content": "[control_753]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 756,
-      "content": "[REFERENCE_DOC_14]",
+      "content": "[control_754]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 757,
-      "content": "[REFERENCE_DOC_13]",
+      "content": "[control_755]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 758,
-      "content": "[REFERENCE_DOC_12]",
+      "content": "[control_756]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 759,
-      "content": "[REFERENCE_DOC_11]",
+      "content": "[control_757]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 760,
-      "content": "[REFERENCE_DOC_10]",
+      "content": "[control_758]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 761,
-      "content": "[REFERENCE_DOC_9]",
+      "content": "[control_759]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 762,
-      "content": "[REFERENCE_DOC_8]",
+      "content": "[control_760]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 763,
-      "content": "[REFERENCE_DOC_7]",
+      "content": "[control_761]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 764,
-      "content": "[REFERENCE_DOC_6]",
+      "content": "[control_762]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 765,
-      "content": "[REFERENCE_DOC_5]",
+      "content": "[control_763]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 766,
-      "content": "[REFERENCE_DOC_4]",
+      "content": "[control_764]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 767,
-      "content": "[REFERENCE_DOC_3]",
+      "content": "[control_765]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 768,
-      "content": "[REFERENCE_DOC_2]",
+      "content": "[control_766]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 769,
-      "content": "[REFERENCE_DOC_1]",
+      "content": "[control_767]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     },
     {
       "id": 770,
-      "content": "[REFERENCE_DOC_0]",
+      "content": "[control_768]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
-      "special": false
+      "special": true
     }
   ],
   "normalizer": {
     "type": "Sequence",
     "normalizers": [
+      {
+        "type": "Prepend",
+        "prepend": "▁"
+      },
       {
         "type": "Replace",
         "pattern": {
@@ -7025,6 +7029,12 @@
       },
       {
         "type": "Fuse"
+      },
+      {
+        "type": "Strip",
+        "content": " ",
+        "start": 1,
+        "stop": 0
       }
     ]
   },
@@ -7048,12 +7058,12 @@
       "[/AVAILABLE_TOOLS]": 7,
       "[TOOL_RESULTS]": 8,
       "[/TOOL_RESULTS]": 9,
-      "[IMG]": 10,
-      "[PREFIX]": 11,
-      "[MIDDLE]": 12,
-      "[SUFFIX]": 13,
-      "[IMG_BREAK]": 14,
-      "[IMG_END]": 15,
+      "[control_8]": 10,
+      "[control_9]": 11,
+      "[control_10]": 12,
+      "[control_11]": 13,
+      "[control_12]": 14,
+      "[control_13]": 15,
       "[control_14]": 16,
       "[control_15]": 17,
       "[control_16]": 18,
@@ -7789,26 +7799,26 @@
       "[control_746]": 748,
       "[control_747]": 749,
       "[control_748]": 750,
-      "[REFERENCE_DOC_19]": 751,
-      "[REFERENCE_DOC_18]": 752,
-      "[REFERENCE_DOC_17]": 753,
-      "[REFERENCE_DOC_16]": 754,
-      "[REFERENCE_DOC_15]": 755,
-      "[REFERENCE_DOC_14]": 756,
-      "[REFERENCE_DOC_13]": 757,
-      "[REFERENCE_DOC_12]": 758,
-      "[REFERENCE_DOC_11]": 759,
-      "[REFERENCE_DOC_10]": 760,
-      "[REFERENCE_DOC_9]": 761,
-      "[REFERENCE_DOC_8]": 762,
-      "[REFERENCE_DOC_7]": 763,
-      "[REFERENCE_DOC_6]": 764,
-      "[REFERENCE_DOC_5]": 765,
-      "[REFERENCE_DOC_4]": 766,
-      "[REFERENCE_DOC_3]": 767,
-      "[REFERENCE_DOC_2]": 768,
-      "[REFERENCE_DOC_1]": 769,
-      "[REFERENCE_DOC_0]": 770,
+      "[control_749]": 751,
+      "[control_750]": 752,
+      "[control_751]": 753,
+      "[control_752]": 754,
+      "[control_753]": 755,
+      "[control_754]": 756,
+      "[control_755]": 757,
+      "[control_756]": 758,
+      "[control_757]": 759,
+      "[control_758]": 760,
+      "[control_759]": 761,
+      "[control_760]": 762,
+      "[control_761]": 763,
+      "[control_762]": 764,
+      "[control_763]": 765,
+      "[control_764]": 766,
+      "[control_765]": 767,
+      "[control_766]": 768,
+      "[control_767]": 769,
+      "[control_768]": 770,
       "<0x00>": 771,
       "<0x01>": 772,
       "<0x02>": 773,

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:59f95e28944c062244741268596badc900df86c7f5ded05088d2da22a7379e06
-size 587583
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -83,7 +84,7 @@
       "special": true
     },
     "10": {
-      "content": "[IMG]",
+      "content": "[control_8]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -91,7 +92,7 @@
       "special": true
     },
     "11": {
-      "content": "[PREFIX]",
+      "content": "[control_9]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -99,7 +100,7 @@
       "special": true
     },
     "12": {
-      "content": "[MIDDLE]",
+      "content": "[control_10]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -107,7 +108,7 @@
       "special": true
     },
     "13": {
-      "content": "[SUFFIX]",
+      "content": "[control_11]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -115,7 +116,7 @@
       "special": true
     },
     "14": {
-      "content": "[IMG_BREAK]",
+      "content": "[control_12]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -123,7 +124,7 @@
       "special": true
     },
     "15": {
-      "content": "[IMG_END]",
+      "content": "[control_13]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -6011,172 +6012,174 @@
       "special": true
     },
     "751": {
-      "content": "[REFERENCE_DOC_19]",
+      "content": "[control_749]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "752": {
-      "content": "[REFERENCE_DOC_18]",
+      "content": "[control_750]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "753": {
-      "content": "[REFERENCE_DOC_17]",
+      "content": "[control_751]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "754": {
-      "content": "[REFERENCE_DOC_16]",
+      "content": "[control_752]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "755": {
-      "content": "[REFERENCE_DOC_15]",
+      "content": "[control_753]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "756": {
-      "content": "[REFERENCE_DOC_14]",
+      "content": "[control_754]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "757": {
-      "content": "[REFERENCE_DOC_13]",
+      "content": "[control_755]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "758": {
-      "content": "[REFERENCE_DOC_12]",
+      "content": "[control_756]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "759": {
-      "content": "[REFERENCE_DOC_11]",
+      "content": "[control_757]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "760": {
-      "content": "[REFERENCE_DOC_10]",
+      "content": "[control_758]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "761": {
-      "content": "[REFERENCE_DOC_9]",
+      "content": "[control_759]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "762": {
-      "content": "[REFERENCE_DOC_8]",
+      "content": "[control_760]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "763": {
-      "content": "[REFERENCE_DOC_7]",
+      "content": "[control_761]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "764": {
-      "content": "[REFERENCE_DOC_6]",
+      "content": "[control_762]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "765": {
-      "content": "[REFERENCE_DOC_5]",
+      "content": "[control_763]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "766": {
-      "content": "[REFERENCE_DOC_4]",
+      "content": "[control_764]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "767": {
-      "content": "[REFERENCE_DOC_3]",
+      "content": "[control_765]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "768": {
-      "content": "[REFERENCE_DOC_2]",
+      "content": "[control_766]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "769": {
-      "content": "[REFERENCE_DOC_1]",
+      "content": "[control_767]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "770": {
-      "content": "[REFERENCE_DOC_0]",
+      "content": "[control_768]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     }
   },
   "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": true,
+  "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_token": "[control_768]",
+  "padding_side": "left",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",