chat-EOS fix: eos_token -> <turn|> (clean stop in generic apps)
Browse files
gpu-pipelined/gemma4_31b_qat_decode_int4linsym_msdpa_g8/tokenizer/tokenizer_config.json
CHANGED
|
@@ -7,7 +7,7 @@
|
|
| 7 |
"eoa_token": "<audio|>",
|
| 8 |
"eoc_token": "<channel|>",
|
| 9 |
"eoi_token": "<image|>",
|
| 10 |
-
"eos_token": "<
|
| 11 |
"eot_token": "<turn|>",
|
| 12 |
"escape_token": "<|\"|>",
|
| 13 |
"etc_token": "<tool_call|>",
|
|
@@ -93,4 +93,4 @@
|
|
| 93 |
"think_token": "<|think|>",
|
| 94 |
"tokenizer_class": "GemmaTokenizer",
|
| 95 |
"unk_token": "<unk>"
|
| 96 |
-
}
|
|
|
|
| 7 |
"eoa_token": "<audio|>",
|
| 8 |
"eoc_token": "<channel|>",
|
| 9 |
"eoi_token": "<image|>",
|
| 10 |
+
"eos_token": "<turn|>",
|
| 11 |
"eot_token": "<turn|>",
|
| 12 |
"escape_token": "<|\"|>",
|
| 13 |
"etc_token": "<tool_call|>",
|
|
|
|
| 93 |
"think_token": "<|think|>",
|
| 94 |
"tokenizer_class": "GemmaTokenizer",
|
| 95 |
"unk_token": "<unk>"
|
| 96 |
+
}
|