Add Open Formosa special tokens

README.md

@@ -11,8 +11,12 @@ tags:
 
 # PangolinTokenizer
 
-Byte-level BPE tokenizer trained for Traditional Chinese, Taiwan text, multilingual
-text, rich transcription, OCR-style text, and generic multimodal control formats.
+Byte-level BPE tokenizer for Traditional Chinese, Taiwan text, multilingual text,
+rich transcription, OCR-style text, and generic control formats.
+
+This revision adds the Open Formosa required control tokens as special tokens.
+The base BPE vocabulary size remains 114,688. The effective tokenizer length,
+including added special tokens, is 114,822.
 
 ## Usage
 
@@ -29,36 +33,12 @@ ids = tokenizer.encode(text)
 decoded = tokenizer.decode(ids)
 ```
 
-## Files
-
-- `config.json`
-- `tokenizer_config.json`
-- `tokenizer.json`
-- `vocab.json`
-- `merges.txt`
-- `special_tokens_map.json`
-- `evaluation_report.json`
+## Open Formosa Compatibility
 
-## Tokenizer Details
-
-- Type: Byte-level BPE
-- Vocabulary size: 114,688
-- Learned merges: 114,397
+- Required special tokens present: 157
+- Required special tokens encode as single IDs: yes
+- Standard special tokens: `<unk>`, `<s>`, `</s>`, `<pad>`
 - Model max length metadata: 131,072
-- Minimum merge frequency: 5
-- Transformers class: `GPT2TokenizerFast`
 - `trust_remote_code`: not required
-
-## Safety Notes
-
-This tokenizer intentionally does not include discrete audio codec token ranges.
-It also intentionally does not include dense timestamp token ranges. Audio should
-be represented through external references or embeddings outside this tokenizer.
-
-Evaluation checks confirmed:
-
-- Transformers `AutoTokenizer` loading works with `trust_remote_code=False`
-- Traditional Chinese and Bopomofo smoke roundtrip works
-- No tokens matching `<|audio_[0-9]+|>`
-- No dense timestamp token ranges matching `<|ts_[0-9]+|>`,
-  `<|timestamp_[0-9]+|>`, or `<|time_[0-9]+|>`
+- No discrete audio codec token ranges are included.
+- No dense timestamp token ranges are included.

added_tokens.json

@@ -0,0 +1,171 @@
+{
+  "<|pad|>": 114653,
+  "<|bos|>": 114654,
+  "<|eos|>": 114655,
+  "<|unk|>": 114656,
+  "<|system|>": 114657,
+  "<|user_channel|>": 114658,
+  "<|assistant_channel|>": 114659,
+  "<|task:speech_to_text|>": 114660,
+  "<|task:text_to_speech|>": 114661,
+  "<|input_audio_start|>": 114662,
+  "<|input_audio_end|>": 114663,
+  "<|audio_ref_start|>": 114664,
+  "<|audio_ref_end|>": 114665,
+  "<|audio_start|>": 114666,
+  "<|audio_end|>": 114667,
+  "<|speech_start|>": 114668,
+  "<|speech_end|>": 114669,
+  "<|transcript_start|>": 114670,
+  "<|transcript_end|>": 114671,
+  "<|segment_start|>": 114672,
+  "<|segment_end|>": 114673,
+  "<|speaker|>": 114674,
+  "<|start_time|>": 114675,
+  "<|end_time|>": 114676,
+  "<|duration|>": 114677,
+  "<|content|>": 114678,
+  "<|non_speech_event|>": 114679,
+  "<|retrieval_result_start|>": 114680,
+  "<|retrieval_result_end|>": 114681,
+  "<|ocr_start|>": 114682,
+  "<|ocr_end|>": 114683,
+  "<|image_start|>": 114684,
+  "<|image_end|>": 114685,
+  "<|video_start|>": 114686,
+  "<|video_end|>": 114687,
+  "<unk>": 114688,
+  "<s>": 114689,
+  "</s>": 114690,
+  "<pad>": 114691,
+  "<|user|>": 114692,
+  "<|assistant|>": 114693,
+  "<|tool_call|>": 114694,
+  "<|tool_response|>": 114695,
+  "<|endoftext|>": 114696,
+  "<think>": 114697,
+  "</think>": 114698,
+  "<|no_think|>": 114699,
+  "<|think|>": 114700,
+  "<|think_max|>": 114701,
+  "<|task:text_to_text|>": 114702,
+  "<|task:speech_to_speech|>": 114703,
+  "<|task:text_speech_to_text|>": 114704,
+  "<|task:text_speech_to_speech|>": 114705,
+  "<|task:full_duplex_speech|>": 114706,
+  "<|task:agent|>": 114707,
+  "<|task:tool_use|>": 114708,
+  "<|task:rag|>": 114709,
+  "<|task:code_execution|>": 114710,
+  "<|task:document_qa|>": 114711,
+  "<|task:data_analysis|>": 114712,
+  "<|task:workflow|>": 114713,
+  "<|reasoning_mode:none|>": 114714,
+  "<|reasoning_mode:short|>": 114715,
+  "<|reasoning_mode:deep|>": 114716,
+  "<|reasoning_mode:verify|>": 114717,
+  "<|private_reasoning_start|>": 114718,
+  "<|private_reasoning_end|>": 114719,
+  "<|reasoning_summary_start|>": 114720,
+  "<|reasoning_summary_end|>": 114721,
+  "<|plan_start|>": 114722,
+  "<|plan_end|>": 114723,
+  "<|step_start|>": 114724,
+  "<|step_end|>": 114725,
+  "<|action_start|>": 114726,
+  "<|action_end|>": 114727,
+  "<|observation_start|>": 114728,
+  "<|observation_end|>": 114729,
+  "<|reflection_start|>": 114730,
+  "<|reflection_end|>": 114731,
+  "<|verification_start|>": 114732,
+  "<|verification_end|>": 114733,
+  "<|tool_schema_start|>": 114734,
+  "<|tool_schema_end|>": 114735,
+  "<|tool_call_start|>": 114736,
+  "<|tool_call_end|>": 114737,
+  "<|tool_result_start|>": 114738,
+  "<|tool_result_end|>": 114739,
+  "<|tool_error_start|>": 114740,
+  "<|tool_error_end|>": 114741,
+  "<|retrieval_query_start|>": 114742,
+  "<|retrieval_query_end|>": 114743,
+  "<|citation_start|>": 114744,
+  "<|citation_end|>": 114745,
+  "<|memory_read_start|>": 114746,
+  "<|memory_read_end|>": 114747,
+  "<|memory_write_start|>": 114748,
+  "<|memory_write_end|>": 114749,
+  "<|final_answer_start|>": 114750,
+  "<|final_answer_end|>": 114751,
+  "<|json_start|>": 114752,
+  "<|json_end|>": 114753,
+  "<|code_start|>": 114754,
+  "<|code_end|>": 114755,
+  "<|markdown_start|>": 114756,
+  "<|markdown_end|>": 114757,
+  "<|duplex_start|>": 114758,
+  "<|duplex_end|>": 114759,
+  "<|system_channel|>": 114760,
+  "<|listen|>": 114761,
+  "<|speak|>": 114762,
+  "<|listen_speak|>": 114763,
+  "<|output_audio_start|>": 114764,
+  "<|output_audio_end|>": 114765,
+  "<|text_start|>": 114766,
+  "<|text_end|>": 114767,
+  "<|overlap|>": 114768,
+  "<|barge_in|>": 114769,
+  "<|interruption|>": 114770,
+  "<|interruption_repair|>": 114771,
+  "<|backchannel|>": 114772,
+  "<|turn_yield|>": 114773,
+  "<|hold|>": 114774,
+  "<|silence|>": 114775,
+  "<|non_speech|>": 114776,
+  "<|voice_reference_start|>": 114777,
+  "<|voice_reference_end|>": 114778,
+  "<|voice_reference|>": 114779,
+  "<|voice_switch|>": 114780,
+  "<|speaker_style|>": 114781,
+  "<|prosody_control|>": 114782,
+  "<|zh_tw|>": 114783,
+  "<|zh_hant|>": 114784,
+  "<|taigi|>": 114785,
+  "<|hakka|>": 114786,
+  "<|bopomofo|>": 114787,
+  "<|mixed_en|>": 114788,
+  "<|en|>": 114789,
+  "<|ja|>": 114790,
+  "<|ko|>": 114791,
+  "<|vi|>": 114792,
+  "<|id|>": 114793,
+  "<|th|>": 114794,
+  "<|asr|>": 114795,
+  "<|tts|>": 114796,
+  "<|speaker_0|>": 114797,
+  "<|speaker_1|>": 114798,
+  "<|speaker_2|>": 114799,
+  "<|speaker_3|>": 114800,
+  "<|timestamp|>": 114801,
+  "<|noise|>": 114802,
+  "<|laugh|>": 114803,
+  "<|breath|>": 114804,
+  "<|pause|>": 114805,
+  "<|prosody|>": 114806,
+  "<|pron|>": 114807,
+  "</|pron|>": 114808,
+  "<|image|>": 114809,
+  "<|ocr|>": 114810,
+  "<|bbox|>": 114811,
+  "<|line|>": 114812,
+  "<|table|>": 114813,
+  "<|row|>": 114814,
+  "<|col|>": 114815,
+  "<|cell|>": 114816,
+  "<|reading_order|>": 114817,
+  "<|source|>": 114818,
+  "<|cite|>": 114819,
+  "<|evidence|>": 114820,
+  "<|quote|>": 114821
+}

evaluation_report.json

@@ -1,58 +1,24 @@
 {
   "ok": true,
-  "failures": [],
-  "sections": {
-    "Timestamp / rich transcription checks": {
-      "rich_transcription_token_ids": {
-        "<|transcript_start|>": 114670,
-        "<|transcript_end|>": 114671,
-        "<|segment_start|>": 114672,
-        "<|segment_end|>": 114673,
-        "<|speaker|>": 114674,
-        "<|start_time|>": 114675,
-        "<|end_time|>": 114676,
-        "<|duration|>": 114677,
-        "<|content|>": 114678,
-        "<|non_speech_event|>": 114679
-      },
-      "timestamp_precision_digits": 2,
-      "json_roundtrip_ok": true,
-      "dense_timestamp_tokens_found": [],
-      "missing_rich_transcription_tokens": [],
-      "timestamp_strings_present": {
-        "0.00": true,
-        "3.42": true,
-        "10.25": true,
-        "3575.50": true
-      },
-      "non_speech_labels_present": {
-        "[Silence]": true,
-        "[Noise]": true,
-        "[Music]": true,
-        "[Unintelligible Speech]": true
-      },
-      "text_roundtrip_ok": {
-        "traditional_chinese": true,
-        "bopomofo_mixed_romanization": true,
-        "json_syntax": true
-      },
-      "required_fields_ok": true,
-      "parse_error": null
-    }
+  "tokenizer": "voidful/PangolinTokenizer",
+  "base_vocab_size": 114688,
+  "effective_vocab_size": 114822,
+  "required_special_token_count": 157,
+  "required_special_tokens_missing": [],
+  "required_special_tokens_single_id": true,
+  "standard_special_tokens": {
+    "bos_token": "<s>",
+    "eos_token": "</s>",
+    "unk_token": "<unk>",
+    "pad_token": "<pad>"
   },
-  "rich_transcription_token_ids": {
-    "<|transcript_start|>": 114670,
-    "<|transcript_end|>": 114671,
-    "<|segment_start|>": 114672,
-    "<|segment_end|>": 114673,
-    "<|speaker|>": 114674,
-    "<|start_time|>": 114675,
-    "<|end_time|>": 114676,
-    "<|duration|>": 114677,
-    "<|content|>": 114678,
-    "<|non_speech_event|>": 114679
+  "banned_token_matches": {
+    "audio_numeric_tokens": [],
+    "dense_timestamp_tokens": []
   },
-  "timestamp_precision_digits": 2,
-  "json_roundtrip_ok": true,
-  "dense_timestamp_tokens_found": []
+  "smoke_texts": {
+    "traditional_chinese": "台灣本土語言模型需要保留繁體中文、注音ㄅㄆㄇ與台語漢字。",
+    "duplex": "<|task:full_duplex_speech|><|audio_ref_start|>audio://utterance_000001<|audio_ref_end|>",
+    "tool_call": "<|tool_call_start|>{\"name\":\"search_documents\",\"arguments\":{\"query\":\"健保年金\"}}<|tool_call_end|>"
+  }
 }

special_tokens_map.json

@@ -1,5 +1,13 @@
 {
+  "unk_token": "<unk>",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
   "additional_special_tokens": [
+    "<|bos|>",
+    "<|eos|>",
+    "<|unk|>",
+    "<|pad|>",
     "<|system|>",
     "<|user_channel|>",
     "<|assistant_channel|>",
@@ -30,10 +38,136 @@
     "<|image_start|>",
     "<|image_end|>",
     "<|video_start|>",
-    "<|video_end|>"
-  ],
-  "pad_token": "<|pad|>",
-  "bos_token": "<|bos|>",
-  "eos_token": "<|eos|>",
-  "unk_token": "<|unk|>"
+    "<|video_end|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|tool_call|>",
+    "<|tool_response|>",
+    "<|endoftext|>",
+    "<think>",
+    "</think>",
+    "<|no_think|>",
+    "<|think|>",
+    "<|think_max|>",
+    "<|task:text_to_text|>",
+    "<|task:speech_to_speech|>",
+    "<|task:text_speech_to_text|>",
+    "<|task:text_speech_to_speech|>",
+    "<|task:full_duplex_speech|>",
+    "<|task:agent|>",
+    "<|task:tool_use|>",
+    "<|task:rag|>",
+    "<|task:code_execution|>",
+    "<|task:document_qa|>",
+    "<|task:data_analysis|>",
+    "<|task:workflow|>",
+    "<|reasoning_mode:none|>",
+    "<|reasoning_mode:short|>",
+    "<|reasoning_mode:deep|>",
+    "<|reasoning_mode:verify|>",
+    "<|private_reasoning_start|>",
+    "<|private_reasoning_end|>",
+    "<|reasoning_summary_start|>",
+    "<|reasoning_summary_end|>",
+    "<|plan_start|>",
+    "<|plan_end|>",
+    "<|step_start|>",
+    "<|step_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|observation_start|>",
+    "<|observation_end|>",
+    "<|reflection_start|>",
+    "<|reflection_end|>",
+    "<|verification_start|>",
+    "<|verification_end|>",
+    "<|tool_schema_start|>",
+    "<|tool_schema_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_result_start|>",
+    "<|tool_result_end|>",
+    "<|tool_error_start|>",
+    "<|tool_error_end|>",
+    "<|retrieval_query_start|>",
+    "<|retrieval_query_end|>",
+    "<|citation_start|>",
+    "<|citation_end|>",
+    "<|memory_read_start|>",
+    "<|memory_read_end|>",
+    "<|memory_write_start|>",
+    "<|memory_write_end|>",
+    "<|final_answer_start|>",
+    "<|final_answer_end|>",
+    "<|json_start|>",
+    "<|json_end|>",
+    "<|code_start|>",
+    "<|code_end|>",
+    "<|markdown_start|>",
+    "<|markdown_end|>",
+    "<|duplex_start|>",
+    "<|duplex_end|>",
+    "<|system_channel|>",
+    "<|listen|>",
+    "<|speak|>",
+    "<|listen_speak|>",
+    "<|output_audio_start|>",
+    "<|output_audio_end|>",
+    "<|text_start|>",
+    "<|text_end|>",
+    "<|overlap|>",
+    "<|barge_in|>",
+    "<|interruption|>",
+    "<|interruption_repair|>",
+    "<|backchannel|>",
+    "<|turn_yield|>",
+    "<|hold|>",
+    "<|silence|>",
+    "<|non_speech|>",
+    "<|voice_reference_start|>",
+    "<|voice_reference_end|>",
+    "<|voice_reference|>",
+    "<|voice_switch|>",
+    "<|speaker_style|>",
+    "<|prosody_control|>",
+    "<|zh_tw|>",
+    "<|zh_hant|>",
+    "<|taigi|>",
+    "<|hakka|>",
+    "<|bopomofo|>",
+    "<|mixed_en|>",
+    "<|en|>",
+    "<|ja|>",
+    "<|ko|>",
+    "<|vi|>",
+    "<|id|>",
+    "<|th|>",
+    "<|asr|>",
+    "<|tts|>",
+    "<|speaker_0|>",
+    "<|speaker_1|>",
+    "<|speaker_2|>",
+    "<|speaker_3|>",
+    "<|timestamp|>",
+    "<|noise|>",
+    "<|laugh|>",
+    "<|breath|>",
+    "<|pause|>",
+    "<|prosody|>",
+    "<|pron|>",
+    "</|pron|>",
+    "<|image|>",
+    "<|ocr|>",
+    "<|bbox|>",
+    "<|line|>",
+    "<|table|>",
+    "<|row|>",
+    "<|col|>",
+    "<|cell|>",
+    "<|reading_order|>",
+    "<|source|>",
+    "<|cite|>",
+    "<|evidence|>",
+    "<|quote|>"
+  ]
 }

tokenizer.json

@@ -317,6 +317,1212 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 114688,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114689,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114690,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114691,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114692,
+      "content": "<|user|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114693,
+      "content": "<|assistant|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114694,
+      "content": "<|tool_call|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114695,
+      "content": "<|tool_response|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114696,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114697,
+      "content": "<think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114698,
+      "content": "</think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114699,
+      "content": "<|no_think|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114700,
+      "content": "<|think|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114701,
+      "content": "<|think_max|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114702,
+      "content": "<|task:text_to_text|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114703,
+      "content": "<|task:speech_to_speech|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114704,
+      "content": "<|task:text_speech_to_text|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114705,
+      "content": "<|task:text_speech_to_speech|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114706,
+      "content": "<|task:full_duplex_speech|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114707,
+      "content": "<|task:agent|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114708,
+      "content": "<|task:tool_use|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114709,
+      "content": "<|task:rag|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114710,
+      "content": "<|task:code_execution|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114711,
+      "content": "<|task:document_qa|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114712,
+      "content": "<|task:data_analysis|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114713,
+      "content": "<|task:workflow|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114714,
+      "content": "<|reasoning_mode:none|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114715,
+      "content": "<|reasoning_mode:short|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114716,
+      "content": "<|reasoning_mode:deep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114717,
+      "content": "<|reasoning_mode:verify|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114718,
+      "content": "<|private_reasoning_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114719,
+      "content": "<|private_reasoning_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114720,
+      "content": "<|reasoning_summary_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114721,
+      "content": "<|reasoning_summary_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114722,
+      "content": "<|plan_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114723,
+      "content": "<|plan_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114724,
+      "content": "<|step_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114725,
+      "content": "<|step_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114726,
+      "content": "<|action_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114727,
+      "content": "<|action_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114728,
+      "content": "<|observation_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114729,
+      "content": "<|observation_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114730,
+      "content": "<|reflection_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114731,
+      "content": "<|reflection_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114732,
+      "content": "<|verification_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114733,
+      "content": "<|verification_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114734,
+      "content": "<|tool_schema_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114735,
+      "content": "<|tool_schema_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114736,
+      "content": "<|tool_call_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114737,
+      "content": "<|tool_call_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114738,
+      "content": "<|tool_result_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114739,
+      "content": "<|tool_result_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114740,
+      "content": "<|tool_error_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114741,
+      "content": "<|tool_error_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114742,
+      "content": "<|retrieval_query_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114743,
+      "content": "<|retrieval_query_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114744,
+      "content": "<|citation_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114745,
+      "content": "<|citation_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114746,
+      "content": "<|memory_read_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114747,
+      "content": "<|memory_read_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114748,
+      "content": "<|memory_write_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114749,
+      "content": "<|memory_write_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114750,
+      "content": "<|final_answer_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114751,
+      "content": "<|final_answer_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114752,
+      "content": "<|json_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114753,
+      "content": "<|json_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114754,
+      "content": "<|code_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114755,
+      "content": "<|code_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114756,
+      "content": "<|markdown_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114757,
+      "content": "<|markdown_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114758,
+      "content": "<|duplex_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114759,
+      "content": "<|duplex_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114760,
+      "content": "<|system_channel|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114761,
+      "content": "<|listen|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114762,
+      "content": "<|speak|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114763,
+      "content": "<|listen_speak|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114764,
+      "content": "<|output_audio_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114765,
+      "content": "<|output_audio_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114766,
+      "content": "<|text_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114767,
+      "content": "<|text_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114768,
+      "content": "<|overlap|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114769,
+      "content": "<|barge_in|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114770,
+      "content": "<|interruption|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114771,
+      "content": "<|interruption_repair|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114772,
+      "content": "<|backchannel|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114773,
+      "content": "<|turn_yield|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114774,
+      "content": "<|hold|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114775,
+      "content": "<|silence|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114776,
+      "content": "<|non_speech|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114777,
+      "content": "<|voice_reference_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114778,
+      "content": "<|voice_reference_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114779,
+      "content": "<|voice_reference|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114780,
+      "content": "<|voice_switch|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114781,
+      "content": "<|speaker_style|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114782,
+      "content": "<|prosody_control|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114783,
+      "content": "<|zh_tw|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114784,
+      "content": "<|zh_hant|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114785,
+      "content": "<|taigi|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114786,
+      "content": "<|hakka|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114787,
+      "content": "<|bopomofo|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114788,
+      "content": "<|mixed_en|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114789,
+      "content": "<|en|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114790,
+      "content": "<|ja|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114791,
+      "content": "<|ko|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114792,
+      "content": "<|vi|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114793,
+      "content": "<|id|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114794,
+      "content": "<|th|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114795,
+      "content": "<|asr|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114796,
+      "content": "<|tts|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114797,
+      "content": "<|speaker_0|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114798,
+      "content": "<|speaker_1|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114799,
+      "content": "<|speaker_2|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114800,
+      "content": "<|speaker_3|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114801,
+      "content": "<|timestamp|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114802,
+      "content": "<|noise|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114803,
+      "content": "<|laugh|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114804,
+      "content": "<|breath|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114805,
+      "content": "<|pause|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114806,
+      "content": "<|prosody|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114807,
+      "content": "<|pron|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114808,
+      "content": "</|pron|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114809,
+      "content": "<|image|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114810,
+      "content": "<|ocr|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114811,
+      "content": "<|bbox|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114812,
+      "content": "<|line|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114813,
+      "content": "<|table|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114814,
+      "content": "<|row|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114815,
+      "content": "<|col|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114816,
+      "content": "<|cell|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114817,
+      "content": "<|reading_order|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114818,
+      "content": "<|source|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114819,
+      "content": "<|cite|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114820,
+      "content": "<|evidence|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 114821,
+      "content": "<|quote|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,
@@ -361,9 +1567,9 @@
   "model": {
     "type": "BPE",
     "dropout": null,
-    "unk_token": "<|unk|>",
-    "continuing_subword_prefix": null,
-    "end_of_word_suffix": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
     "fuse_unk": false,
     "byte_fallback": false,
     "ignore_merges": false,

tokenizer_config.json

@@ -1,8 +1,15 @@
 {
+  "add_prefix_space": false,
   "backend": "tokenizers",
-  "bos_token": "<|bos|>",
-  "eos_token": "<|eos|>",
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "errors": "replace",
   "extra_special_tokens": [
+    "<|bos|>",
+    "<|eos|>",
+    "<|unk|>",
+    "<|pad|>",
     "<|system|>",
     "<|user_channel|>",
     "<|assistant_channel|>",
@@ -33,11 +40,147 @@
     "<|image_start|>",
     "<|image_end|>",
     "<|video_start|>",
-    "<|video_end|>"
+    "<|video_end|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|tool_call|>",
+    "<|tool_response|>",
+    "<|endoftext|>",
+    "<think>",
+    "</think>",
+    "<|no_think|>",
+    "<|think|>",
+    "<|think_max|>",
+    "<|task:text_to_text|>",
+    "<|task:speech_to_speech|>",
+    "<|task:text_speech_to_text|>",
+    "<|task:text_speech_to_speech|>",
+    "<|task:full_duplex_speech|>",
+    "<|task:agent|>",
+    "<|task:tool_use|>",
+    "<|task:rag|>",
+    "<|task:code_execution|>",
+    "<|task:document_qa|>",
+    "<|task:data_analysis|>",
+    "<|task:workflow|>",
+    "<|reasoning_mode:none|>",
+    "<|reasoning_mode:short|>",
+    "<|reasoning_mode:deep|>",
+    "<|reasoning_mode:verify|>",
+    "<|private_reasoning_start|>",
+    "<|private_reasoning_end|>",
+    "<|reasoning_summary_start|>",
+    "<|reasoning_summary_end|>",
+    "<|plan_start|>",
+    "<|plan_end|>",
+    "<|step_start|>",
+    "<|step_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|observation_start|>",
+    "<|observation_end|>",
+    "<|reflection_start|>",
+    "<|reflection_end|>",
+    "<|verification_start|>",
+    "<|verification_end|>",
+    "<|tool_schema_start|>",
+    "<|tool_schema_end|>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_result_start|>",
+    "<|tool_result_end|>",
+    "<|tool_error_start|>",
+    "<|tool_error_end|>",
+    "<|retrieval_query_start|>",
+    "<|retrieval_query_end|>",
+    "<|citation_start|>",
+    "<|citation_end|>",
+    "<|memory_read_start|>",
+    "<|memory_read_end|>",
+    "<|memory_write_start|>",
+    "<|memory_write_end|>",
+    "<|final_answer_start|>",
+    "<|final_answer_end|>",
+    "<|json_start|>",
+    "<|json_end|>",
+    "<|code_start|>",
+    "<|code_end|>",
+    "<|markdown_start|>",
+    "<|markdown_end|>",
+    "<|duplex_start|>",
+    "<|duplex_end|>",
+    "<|system_channel|>",
+    "<|listen|>",
+    "<|speak|>",
+    "<|listen_speak|>",
+    "<|output_audio_start|>",
+    "<|output_audio_end|>",
+    "<|text_start|>",
+    "<|text_end|>",
+    "<|overlap|>",
+    "<|barge_in|>",
+    "<|interruption|>",
+    "<|interruption_repair|>",
+    "<|backchannel|>",
+    "<|turn_yield|>",
+    "<|hold|>",
+    "<|silence|>",
+    "<|non_speech|>",
+    "<|voice_reference_start|>",
+    "<|voice_reference_end|>",
+    "<|voice_reference|>",
+    "<|voice_switch|>",
+    "<|speaker_style|>",
+    "<|prosody_control|>",
+    "<|zh_tw|>",
+    "<|zh_hant|>",
+    "<|taigi|>",
+    "<|hakka|>",
+    "<|bopomofo|>",
+    "<|mixed_en|>",
+    "<|en|>",
+    "<|ja|>",
+    "<|ko|>",
+    "<|vi|>",
+    "<|id|>",
+    "<|th|>",
+    "<|asr|>",
+    "<|tts|>",
+    "<|speaker_0|>",
+    "<|speaker_1|>",
+    "<|speaker_2|>",
+    "<|speaker_3|>",
+    "<|timestamp|>",
+    "<|noise|>",
+    "<|laugh|>",
+    "<|breath|>",
+    "<|pause|>",
+    "<|prosody|>",
+    "<|pron|>",
+    "</|pron|>",
+    "<|image|>",
+    "<|ocr|>",
+    "<|bbox|>",
+    "<|line|>",
+    "<|table|>",
+    "<|row|>",
+    "<|col|>",
+    "<|cell|>",
+    "<|reading_order|>",
+    "<|source|>",
+    "<|cite|>",
+    "<|evidence|>",
+    "<|quote|>"
   ],
+  "fix_mistral_regex": true,
+  "is_local": false,
+  "local_files_only": false,
   "model_max_length": 131072,
   "model_type": "byte_level_bpe",
-  "pad_token": "<|pad|>",
+  "no_audio_codec_tokens": true,
+  "no_dense_timestamp_tokens": true,
+  "pad_token": "<pad>",
+  "padding_side": "right",
   "rich_transcription": {
     "allow_non_speech_events": true,
     "compact_json": true,
@@ -88,13 +231,20 @@
     "<|video_end|>"
   ],
   "strict_no_dense_timestamp_tokens": true,
-  "tokenizer_class": "GPT2TokenizerFast",
-  "unk_token": "<|unk|>",
-  "vocab_size": 114688,
-  "padding_side": "right",
+  "tokenizer_class": "GPT2Tokenizer",
   "truncation_side": "right",
-  "clean_up_tokenization_spaces": false,
-  "no_audio_codec_tokens": true,
-  "no_dense_timestamp_tokens": true,
-  "fix_mistral_regex": true
+  "unk_token": "<unk>",
+  "vocab_size": 114688,
+  "effective_vocab_size": 114822,
+  "open_formosa": {
+    "required_special_token_count": 157,
+    "required_special_tokens_present": true,
+    "required_special_tokens_single_id": true,
+    "standard_special_tokens": {
+      "unk_token": "<unk>",
+      "bos_token": "<s>",
+      "eos_token": "</s>",
+      "pad_token": "<pad>"
+    }
+  }
 }