Tokenizer for checkpoint at step 5000 (eval_loss=0.5764)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,45 @@
+[gMASK]<sop>
+{%- if tools -%}
+<|system|>
+你是一个名为 ChatGLM 的人工智能助手。你是基于智谱 AI 公司训练的语言模型 GLM-4 模型开发的，你的任务是针对用户的问题和要求提供适当的答复和支持。
+
+# 可用工具
+{%- for tool in tools %}
+    {%- set function = tool.function if tool.get("function") else tool %}
+
+## {{ function.name }}
+
+{{ function | tojson(indent=4, ensure_ascii=False) }}
+在调用上述函数时，请使用 Json 格式表示调用的参数。
+{%- endfor %}
+{%- endif -%}
+
+{%- for msg in messages %}
+    {%- if msg.role == 'system' %}
+<|system|>
+{{ msg.content }}
+    {%- endif %}
+{%- endfor %}
+
+{%- for message in messages if message.role != 'system' %}
+    {%- set role = message['role'] %}
+    {%- set content = message['content'] %}
+    {%- set visible = content.split('</think>')[-1].strip() %}
+    {%- set meta = message.get("metadata", "") %}
+
+    {%- if role == 'user' %}
+<|user|>
+{{ visible }}
+    {%- elif role == 'assistant' and not meta %}
+<|assistant|>
+{{ visible }}
+    {%- elif role == 'assistant' and meta %}
+<|assistant|>{{ meta }}
+{{ visible }}
+    {%- elif role == 'observation' %}
+<|observation|>
+{{ visible }}
+    {%- endif %}
+{%- endfor %}
+{% if add_generation_prompt %}<|assistant|>
+<think>{% endif %}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76ebeac0d8bd7879ead7b43c16b44981f277e47225de2bd7de9ae1a6cc664a8c
+size 19966496

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "is_local": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 128000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "TokenizersBackend"
+}