mlboydaisuke commited on
Commit
64a12f1
·
verified ·
1 Parent(s): 68f492a

chat-EOS fix: eos_token -> <turn|> (clean stop in generic apps)

Browse files
gpu-pipelined/gemma4_31b_qat_decode_int4linsym_msdpa_g8/tokenizer/tokenizer_config.json CHANGED
@@ -7,7 +7,7 @@
7
  "eoa_token": "<audio|>",
8
  "eoc_token": "<channel|>",
9
  "eoi_token": "<image|>",
10
- "eos_token": "<eos>",
11
  "eot_token": "<turn|>",
12
  "escape_token": "<|\"|>",
13
  "etc_token": "<tool_call|>",
@@ -93,4 +93,4 @@
93
  "think_token": "<|think|>",
94
  "tokenizer_class": "GemmaTokenizer",
95
  "unk_token": "<unk>"
96
- }
 
7
  "eoa_token": "<audio|>",
8
  "eoc_token": "<channel|>",
9
  "eoi_token": "<image|>",
10
+ "eos_token": "<turn|>",
11
  "eot_token": "<turn|>",
12
  "escape_token": "<|\"|>",
13
  "etc_token": "<tool_call|>",
 
93
  "think_token": "<|think|>",
94
  "tokenizer_class": "GemmaTokenizer",
95
  "unk_token": "<unk>"
96
+ }