mlboydaisuke commited on
Commit
2c38bf9
·
verified ·
1 Parent(s): d619e32

chat-EOS fix: eos_token -> <turn|> (clean stop in generic apps)

Browse files
gpu-pipelined/gemma4_12b_qat_decode_int4linsym_msdpa_g8/tokenizer/tokenizer_config.json CHANGED
@@ -7,7 +7,7 @@
7
  "eoa_token": "<audio|>",
8
  "eoc_token": "<channel|>",
9
  "eoi_token": "<image|>",
10
- "eos_token": "<eos>",
11
  "eot_token": "<turn|>",
12
  "escape_token": "<|\"|>",
13
  "etc_token": "<tool_call|>",
@@ -71,4 +71,4 @@
71
  "think_token": "<|think|>",
72
  "tokenizer_class": "GemmaTokenizer",
73
  "unk_token": "<unk>"
74
- }
 
7
  "eoa_token": "<audio|>",
8
  "eoc_token": "<channel|>",
9
  "eoi_token": "<image|>",
10
+ "eos_token": "<turn|>",
11
  "eot_token": "<turn|>",
12
  "escape_token": "<|\"|>",
13
  "etc_token": "<tool_call|>",
 
71
  "think_token": "<|think|>",
72
  "tokenizer_class": "GemmaTokenizer",
73
  "unk_token": "<unk>"
74
+ }