Upload tokenizer

README.md

@@ -1,11 +1,11 @@
 ---
 language:
-  - en
-  - ja
+- en
+- ja
+license: apache-2.0
 library_name: transformers
 pipeline_tag: text-generation
 model_type: mistral
-license: apache-2.0
 ---
 
 # Swallow-MS-7b-v0.1

special_tokens_map.json

@@ -1,5 +1,23 @@
 {
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "unk_token": "<unk>"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -27,17 +27,21 @@
       "special": true
     }
   },
-  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true and not '<<SYS>>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'あなたは誠実で優秀な日本人のアシスタントです。' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{{ bos_token }}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST] ' }}{% elif message['role'] == 'system' %}{{ '<<SYS>>\\n' + content.strip() + '\\n<</SYS>>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ''  + content.strip() + '' + eos_token }}{% endif %}{% endfor %}",
   "additional_special_tokens": [],
   "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}\n{% set loop_messages = messages[1:] %}\n{% set system_message = messages[0]['content'] %}\n{% elif false == true and not '<<SYS>>' in messages[0]['content'] %}\n{% set loop_messages = messages %}\n{% set system_message = 'あなたは誠実で優秀な日本人のアシスタントです。' %}\n{% else %}\n{% set loop_messages = messages %}\n{% set system_message = false %}\n{% endif %}\n{{ bos_token }}\n{% for message in loop_messages %}\n{% if (message['role'] == 'user') != ((loop.index0 + messages[0]['role'] == 'assistant') % 2 == 0) %}\n{{ raise_exception('Conversation roles must alternate starting from the first role.') }}\n{% endif %}\n{% if loop.index0 == 0 and system_message != false %}\n{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}\n{% else %}\n{% set content = message['content'] %}\n{% endif %}\n{% if message['role'] == 'user' %}\n{{ '[INST] ' + content.strip() + ' [/INST] ' }}\n{% elif message['role'] == 'system' %}\n{{ '<<SYS>>\n' + content.strip() + '\n<</SYS>>\n\n' }}\n{% elif message['role'] == 'assistant' %}\n{{ ''  + content.strip() + '' + eos_token }}\n{% endif %}\n{% endfor %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,
+  "max_length": 4096,
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
+  "stride": 0,
   "tokenizer_class": "LlamaTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
   "unk_token": "<unk>",
   "use_default_system_prompt": false
 }