Upload tokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,45 @@
+
+{%- for message in messages %}
+    {#-- Validate role is a stringified integer --#}
+    {%- if not message['role'] is string or not message['role'].isdigit() %}
+        {{- raise_exception("The role must be an integer or a stringified integer (e.g. '0') designating the speaker id") }}
+    {%- endif %}
+
+    {#-- Validate content is a list --#}
+    {%- set content = message['content'] %}
+    {%- if content is not iterable or content is string %}
+        {{- raise_exception("The content must be a list") }}
+    {%- endif %}
+
+    {#-- Collect content types --#}
+    {%- set content_types = content | map(attribute='type') | list %}
+    {%- set is_last = loop.last %}
+
+    {#-- Last message validation --#}
+    {%- if is_last %}
+        {%- if 'text' not in content_types %}
+            {{- raise_exception("The last message must include one item of type 'text'") }}
+        {%- elif (content_types | select('equalto', 'text') | list | length > 1) or (content_types | select('equalto', 'audio') | list | length > 1) %}
+            {{- raise_exception("At most two items are allowed in the last message: one 'text' and one 'audio'") }}
+        {%- endif %}
+
+    {#-- All other messages validation --#}
+    {%- else %}
+        {%- if content_types | select('equalto', 'text') | list | length != 1
+              or content_types | select('equalto', 'audio') | list | length != 1 %}
+            {{- raise_exception("Each message (except the last) must contain exactly one 'text' and one 'audio' item") }}
+        {%- elif content_types | reject('in', ['text', 'audio']) | list | length > 0 %}
+            {{- raise_exception("Only 'text' and 'audio' types are allowed in content") }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+
+{%- for message in messages %}
+    {{- bos_token }}
+    {{- '[' + message['role'] + ']' }}
+    {{- message['content'][0]['text'] }}
+    {{- eos_token }}
+    {%- if message['content']|length > 1 %}
+        {{- '<|AUDIO|><|audio_eos|>' }}
+    {%- endif %}
+{%- endfor %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f511469b4aceda8c17ccafef2310eceeda9dd4589cf2c062bd0279eef5646d
+size 17209980

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "model_specific_special_tokens": {},
+  "pad_token": "<|end_of_text|>",
+  "processor_class": "CsmProcessor",
+  "tokenizer_class": "TokenizersBackend"
+}