Update to simple question/answer pretraining template (all tokens trainable)

chat_template.jinja

@@ -1,211 +1,11 @@
-{% macro render_extra_keys(json_dict, handled_keys) %}
-    {%- if json_dict is mapping %}
-        {%- for json_key in json_dict if json_key not in handled_keys %}
-            {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
-                {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
-            {%- else %}
-                {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
-            {%- endif %}
-        {%- endfor %}
-    {%- endif %}
-{% endmacro %}
-{%- set kw = chat_template_kwargs if chat_template_kwargs is defined else {} %}
-{%- set enable_thinking = kw.enable_thinking if kw.enable_thinking is defined else True %}
-{%- set reasoning_budget = kw.reasoning_budget if kw.reasoning_budget is defined else None %}
-{%- set truncate_history_thinking = kw.truncate_history_thinking if kw.truncate_history_thinking is defined else True %}
-{%- set response_format = kw.response_format if kw.response_format is defined else None %}
-
-{%- set ns = namespace(last_user_idx = -1) %}
-{%- set loop_messages = messages %}
-{%- for m in loop_messages %}
-  {%- if m["role"] == "user" %}
-    {%- set ns.last_user_idx = loop.index0 %}
-  {%- endif %}
-{%- endfor %}
-
-{%- if messages[0]["role"] == "system" %}
-    {%- set system_message = messages[0]["content"] %}
-    {%- set loop_messages = messages[1:] %}
-{%- else %}
-    {%- set system_message = "" %}
-    {%- set loop_messages = messages %}
-{%- endif %}
-{%- if not tools is defined %}
-    {%- set tools = [] %}
+{#- Simple pretraining chat template: question/answer format, all tokens trainable.
+    No chat-ML markers, no thinking tags, no loss masking. -#}
+{%- for message in messages %}
+{%- if message.role == "system" %}
+{{- message.content }}
+{%- elif message.role == "user" %}
+{{- "\nquestion: " + message.content }}
+{%- elif message.role == "assistant" %}
+{{- "\nanswer: " + message.content }}
 {%- endif %}
-{# Recompute last_user_idx relative to loop_messages after handling system #}
-{%- set ns = namespace(last_user_idx = -1) %}
-{%- for m in loop_messages %}
-  {%- if m["role"] == "user" %}
-    {%- set ns.last_user_idx = loop.index0 %}
-  {%- endif %}
 {%- endfor %}
-{%- if system_message is defined %}
-    {{- "<|im_start|>system\n" + system_message }}
-{%- else %}
-    {%- if tools is iterable and tools | length > 0 %}
-        {{- "<|im_start|>system\n" }}
-    {%- endif %}
-{%- endif %}
-{%- if tools is iterable and tools | length > 0 %}
-    {%- if system_message is defined and system_message | length > 0 %}
-        {{- "\n\n" }}
-    {%- endif %}
-    {{- "# Tools\n\nYou have access to the following functions:\n\n" }}
-    {{- "<tools>" }}
-    {%- for tool in tools %}
-        {%- if tool.function is defined %}
-            {%- set tool = tool.function %}
-        {%- endif %}
-        {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
-        {%- if tool.description is defined %}
-            {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
-        {%- endif %}
-        {{- '\n<parameters>' }}
-        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
-            {%- for param_name, param_fields in tool.parameters.properties|items %}
-                {{- '\n<parameter>' }}
-                {{- '\n<name>' ~ param_name ~ '</name>' }}
-                {%- if param_fields.type is defined %}
-                    {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
-                {%- endif %}
-                {%- if param_fields.description is defined %}
-                    {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
-                {%- endif %}
-                {%- if param_fields.enum is defined %}
-                    {{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
-                {%- endif %}
-                {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
-                {{- render_extra_keys(param_fields, handled_keys) }}
-                {{- '\n</parameter>' }}
-            {%- endfor %}
-        {%- endif %}
-        {% set handled_keys = ['type', 'properties', 'required'] %}
-        {{- render_extra_keys(tool.parameters, handled_keys) }}
-        {%- if tool.parameters is defined and tool.parameters.required is defined %}
-            {{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
-        {%- endif %}
-        {{- '\n</parameters>' }}
-        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
-        {{- render_extra_keys(tool, handled_keys) }}
-        {{- '\n</function>' }}
-    {%- endfor %}
-    {{- "\n</tools>" }}
-
-    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
-{%- endif %}
-
-
-{%- if system_message is defined %}
-    {{- '<|im_end|>\n' }}
-{%- else %}
-    {%- if tools is iterable and tools | length > 0 %}
-        {{- '<|im_end|>\n' }}
-    {%- endif %}
-{%- endif %}
-
-{%- for message in loop_messages %}
-    {%- if message.role == "assistant" %}
-        {# Add reasoning content in to content field for unified processing below. #}
-        {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
-            {%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
-        {%- else %}
-            {%- set content = message.content | default('', true) %}
-            {%- if content is string -%}
-                {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
-                {%- if '<think>' not in content and '</think>' not in content -%}
-                    {%- set content = "<think></think>" ~ content -%}
-                {%- endif -%}
-            {%- else -%}
-                {%- set content = content -%}
-            {%- endif -%}
-        {%- endif %}
-        {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
-            {# Assistant message has tool calls. #}
-            {{- '<|im_start|>assistant\n' }}{% generation %}
-                {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
-                {%- if content is string and content | trim | length > 0 %}
-                    {%- if include_content %}
-                        {{- (content | trim) ~ '\n' -}}
-                    {%- else %}
-                        {%- set c = (content | string) %}
-                        {%- if '</think>' in c %}
-                            {# Keep only content after the last closing think. Also generation prompt causes this. #}
-                            {%- set c = c.split('</think>')[-1] %}
-                        {%- elif '<think>' in c %}
-                            {# If <think> was opened but never closed, drop the trailing think segment #}
-                            {%- set c = c.split('<think>')[0] %}
-                        {%- endif %}
-                        {%- set c = "<think></think>" ~ c | trim %}
-                        {%- if c | length > 0 %}
-                            {{- c ~ '\n' -}}
-                        {%- endif %}
-                    {%- endif %}
-                {%- else %}
-                    {{- "<think></think>" -}}
-                {%- endif %}
-                {%- for tool_call in message.tool_calls %}
-                    {%- if tool_call.function is defined %}
-                        {%- set tool_call = tool_call.function %}
-                    {%- endif %}
-                    {{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
-                        {%- if tool_call.arguments is defined %}
-                            {%- for args_name, args_value in tool_call.arguments|items %}
-                                {{- '<parameter=' ~ args_name ~ '>\n' -}}
-                                    {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
-                                {{- args_value ~ '\n</parameter>\n' -}}
-                            {%- endfor %}
-                        {%- endif %}
-                    {{- '</function>\n</tool_call>\n' -}}
-                {%- endfor %}
-                {{- '<|im_end|>' }}{% endgeneration %}{{- '\n' }}
-        {%- else %}
-            {# Assistant message doesn't have tool calls. #}
-            {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
-                {{- '<|im_start|>assistant\n' }}{% generation %}{{- (content | default('', true) | string | trim) }}{{- '<|im_end|>' }}{% endgeneration %}{{- '\n' }}
-            {%- else %}
-                {%- set c = (content | default('', true) | string) %}
-                {%- if '<think>' in c and '</think>' in c %}
-                    {%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
-                {%- endif %}
-                {%- set c = c | trim %}
-                {%- if c | length > 0 %}
-                    {{- '<|im_start|>assistant\n' }}{% generation %}{{- c }}{{- '<|im_end|>' }}{% endgeneration %}{{- '\n' }}
-                {%- else %}
-                    {{- '<|im_start|>assistant\n' }}{% generation %}{{- '<|im_end|>' }}{% endgeneration %}{{- '\n' }}
-                {%- endif %}
-            {%- endif %}
-        {%- endif %}
-    {%- elif message.role == "user" or message.role == "system" %}
-        {{- '<|im_start|>' + message.role + '\n' }}
-        {%- set content = message.content | string %}
-        {%- if message.role == "user" and loop.index0 == ns.last_user_idx and reasoning_budget is not none %}
-            {{- content + '\n\n{thinking token budget: ' + (reasoning_budget | string) + '}' }}
-        {%- else %}
-            {{- content }}
-        {%- endif %}
-        {{- '<|im_end|>\n' }}
-    {%- elif message.role == "tool" %}
-        {%- if loop.previtem and loop.previtem.role != "tool" %}
-            {{- '<|im_start|>user\n' }}
-        {%- endif %}
-        {{- '<tool_response>\n' }}
-        {{- message.content }}
-        {{- '\n</tool_response>\n' }}
-        {%- if not loop.last and loop.nextitem.role != "tool" %}
-            {{- '<|im_end|>\n' }}
-        {%- elif loop.last %}
-            {{- '<|im_end|>\n' }}
-        {%- endif %}
-    {%- else %}
-        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
-    {%- endif %}
-{%- endfor %}
-
-{%- if add_generation_prompt %}
-    {%- if enable_thinking %}
-        {{- '<|im_start|>assistant\n<think>\n' }}
-    {%- else %}
-        {{- '<|im_start|>assistant\n<think></think>' }}
-    {%- endif %}
-{%- endif %}

tokenizer_config.json

@@ -8015,5 +8015,5 @@
   "model_max_length": 262144,
   "tokenizer_class": "PreTrainedTokenizerFast",
   "unk_token": "<unk>",
-  "chat_template": "{% macro render_extra_keys(json_dict, handled_keys) %}\n    {%- if json_dict is mapping %}\n        {%- for json_key in json_dict if json_key not in handled_keys %}\n            {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}\n                {{- '\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}\n            {%- else %}\n                {{-'\\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}\n            {%- endif %}\n        {%- endfor %}\n    {%- endif %}\n{% endmacro %}\n{%- set kw = chat_template_kwargs if chat_template_kwargs is defined else {} %}\n{%- set enable_thinking = kw.enable_thinking if kw.enable_thinking is defined else True %}\n{%- set reasoning_budget = kw.reasoning_budget if kw.reasoning_budget is defined else None %}\n{%- set truncate_history_thinking = kw.truncate_history_thinking if kw.truncate_history_thinking is defined else True %}\n{%- set response_format = kw.response_format if kw.response_format is defined else None %}\n\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- set loop_messages = messages %}\n{%- for m in loop_messages %}\n  {%- if m[\"role\"] == \"user\" %}\n    {%- set ns.last_user_idx = loop.index0 %}\n  {%- endif %}\n{%- endfor %}\n\n{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = [] %}\n{%- endif %}\n{# Recompute last_user_idx relative to loop_messages after handling system #}\n{%- set ns = namespace(last_user_idx = -1) %}\n{%- for m in loop_messages %}\n  {%- if m[\"role\"] == \"user\" %}\n    {%- set ns.last_user_idx = loop.index0 %}\n  {%- endif %}\n{%- endfor %}\n{%- if system_message is defined %}\n    {{- \"<|im_start|>system\\n\" + system_message }}\n{%- else %}\n    {%- if tools is iterable and tools | length > 0 %}\n        {{- \"<|im_start|>system\\n\" }}\n    {%- endif %}\n{%- endif %}\n{%- if tools is iterable and tools | length > 0 %}\n    {%- if system_message is defined and system_message | length > 0 %}\n        {{- \"\\n\\n\" }}\n    {%- endif %}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n    {{- \"<tools>\" }}\n    {%- for tool in tools %}\n        {%- if tool.function is defined %}\n            {%- set tool = tool.function %}\n        {%- endif %}\n        {{- \"\\n<function>\\n<name>\" ~ tool.name ~ \"</name>\" }}\n        {%- if tool.description is defined %}\n            {{- '\\n<description>' ~ (tool.description | trim) ~ '</description>' }}\n        {%- endif %}\n        {{- '\\n<parameters>' }}\n        {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}\n            {%- for param_name, param_fields in tool.parameters.properties|items %}\n                {{- '\\n<parameter>' }}\n                {{- '\\n<name>' ~ param_name ~ '</name>' }}\n                {%- if param_fields.type is defined %}\n                    {{- '\\n<type>' ~ (param_fields.type | string) ~ '</type>' }}\n                {%- endif %}\n                {%- if param_fields.description is defined %}\n                    {{- '\\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}\n                {%- endif %}\n                {%- if param_fields.enum is defined %}\n                    {{- '\\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}\n                {%- endif %}\n                {%- set handled_keys = ['name', 'type', 'description', 'enum'] %}\n                {{- render_extra_keys(param_fields, handled_keys) }}\n                {{- '\\n</parameter>' }}\n            {%- endfor %}\n        {%- endif %}\n        {% set handled_keys = ['type', 'properties', 'required'] %}\n        {{- render_extra_keys(tool.parameters, handled_keys) }}\n        {%- if tool.parameters is defined and tool.parameters.required is defined %}\n            {{- '\\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}\n        {%- endif %}\n        {{- '\\n</parameters>' }}\n        {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}\n        {{- render_extra_keys(tool, handled_keys) }}\n        {{- '\\n</function>' }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n{%- endif %}\n\n\n{%- if system_message is defined %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if tools is iterable and tools | length > 0 %}\n        {{- '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n\n{%- for message in loop_messages %}\n    {%- if message.role == \"assistant\" %}\n        {# Add reasoning content in to content field for unified processing below. #}\n        {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}\n            {%- set content = \"<think>\\n\" ~ message.reasoning_content ~ \"\\n</think>\\n\" ~ (message.content | default('', true)) %}\n        {%- else %}\n            {%- set content = message.content | default('', true) %}\n            {%- if content is string -%}\n                {# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}\n                {%- if '<think>' not in content and '</think>' not in content -%}\n                    {%- set content = \"<think></think>\" ~ content -%}\n                {%- endif -%}\n            {%- else -%}\n                {%- set content = content -%}\n            {%- endif -%}\n        {%- endif %}\n        {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}\n            {# Assistant message has tool calls. #}\n            {{- '<|im_start|>assistant\\n' }}{% generation %}\n                {%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n                {%- if content is string and content | trim | length > 0 %}\n                    {%- if include_content %}\n                        {{- (content | trim) ~ '\\n' -}}\n                    {%- else %}\n                        {%- set c = (content | string) %}\n                        {%- if '</think>' in c %}\n                            {# Keep only content after the last closing think. Also generation prompt causes this. #}\n                            {%- set c = c.split('</think>')[-1] %}\n                        {%- elif '<think>' in c %}\n                            {# If <think> was opened but never closed, drop the trailing think segment #}\n                            {%- set c = c.split('<think>')[0] %}\n                        {%- endif %}\n                        {%- set c = \"<think></think>\" ~ c | trim %}\n                        {%- if c | length > 0 %}\n                            {{- c ~ '\\n' -}}\n                        {%- endif %}\n                    {%- endif %}\n                {%- else %}\n                    {{- \"<think></think>\" -}}\n                {%- endif %}\n                {%- for tool_call in message.tool_calls %}\n                    {%- if tool_call.function is defined %}\n                        {%- set tool_call = tool_call.function %}\n                    {%- endif %}\n                    {{- '<tool_call>\\n<function=' ~ tool_call.name ~ '>\\n' -}}\n                        {%- if tool_call.arguments is defined %}\n                            {%- for args_name, args_value in tool_call.arguments|items %}\n                                {{- '<parameter=' ~ args_name ~ '>\\n' -}}\n                                    {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n                                {{- args_value ~ '\\n</parameter>\\n' -}}\n                            {%- endfor %}\n                        {%- endif %}\n                    {{- '</function>\\n</tool_call>\\n' -}}\n                {%- endfor %}\n                {{- '<|im_end|>' }}{% endgeneration %}{{- '\\n' }}\n        {%- else %}\n            {# Assistant message doesn't have tool calls. #}\n            {%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}\n                {{- '<|im_start|>assistant\\n' }}{% generation %}{{- (content | default('', true) | string | trim) }}{{- '<|im_end|>' }}{% endgeneration %}{{- '\\n' }}\n            {%- else %}\n                {%- set c = (content | default('', true) | string) %}\n                {%- if '<think>' in c and '</think>' in c %}\n                    {%- set c = \"<think></think>\" ~ c.split('</think>')[-1] %}\n                {%- endif %}\n                {%- set c = c | trim %}\n                {%- if c | length > 0 %}\n                    {{- '<|im_start|>assistant\\n' }}{% generation %}{{- c }}{{- '<|im_end|>' }}{% endgeneration %}{{- '\\n' }}\n                {%- else %}\n                    {{- '<|im_start|>assistant\\n' }}{% generation %}{{- '<|im_end|>' }}{% endgeneration %}{{- '\\n' }}\n                {%- endif %}\n            {%- endif %}\n        {%- endif %}\n    {%- elif message.role == \"user\" or message.role == \"system\" %}\n        {{- '<|im_start|>' + message.role + '\\n' }}\n        {%- set content = message.content | string %}\n        {%- if message.role == \"user\" and loop.index0 == ns.last_user_idx and reasoning_budget is not none %}\n            {{- content + '\\n\\n{thinking token budget: ' + (reasoning_budget | string) + '}' }}\n        {%- else %}\n            {{- content }}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user\\n' }}\n        {%- endif %}\n        {{- '<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>\\n' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n    {%- if enable_thinking %}\n        {{- '<|im_start|>assistant\\n<think>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>assistant\\n<think></think>' }}\n    {%- endif %}\n{%- endif %}\n"
+  "chat_template": "{#- Simple pretraining chat template: question/answer format, all tokens trainable.\n    No chat-ML markers, no thinking tags, no loss masking. -#}\n{%- for message in messages %}\n{%- if message.role == \"system\" %}\n{{- message.content }}\n{%- elif message.role == \"user\" %}\n{{- \"\\nquestion: \" + message.content }}\n{%- elif message.role == \"assistant\" %}\n{{- \"\\nanswer: \" + message.content }}\n{%- endif %}\n{%- endfor %}\n"
 }