Upload Pangolin tokenizer

README.md

@@ -0,0 +1,64 @@
+---
+license: other
+library_name: transformers
+tags:
+  - tokenizer
+  - byte-level-bpe
+  - traditional-chinese
+  - taiwan
+  - multilingual
+---
+
+# PangolinTokenizer
+
+Byte-level BPE tokenizer trained for Traditional Chinese, Taiwan text, multilingual
+text, rich transcription, OCR-style text, and generic multimodal control formats.
+
+## Usage
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "voidful/PangolinTokenizer",
+    trust_remote_code=False,
+)
+
+text = "<|system|>台灣健保與注音ㄅㄆㄇ，Tailo: Tâi-uân"
+ids = tokenizer.encode(text)
+decoded = tokenizer.decode(ids)
+```
+
+## Files
+
+- `config.json`
+- `tokenizer_config.json`
+- `tokenizer.json`
+- `vocab.json`
+- `merges.txt`
+- `special_tokens_map.json`
+- `evaluation_report.json`
+
+## Tokenizer Details
+
+- Type: Byte-level BPE
+- Vocabulary size: 114,688
+- Learned merges: 114,397
+- Model max length metadata: 131,072
+- Minimum merge frequency: 5
+- Transformers class: `GPT2TokenizerFast`
+- `trust_remote_code`: not required
+
+## Safety Notes
+
+This tokenizer intentionally does not include discrete audio codec token ranges.
+It also intentionally does not include dense timestamp token ranges. Audio should
+be represented through external references or embeddings outside this tokenizer.
+
+Evaluation checks confirmed:
+
+- Transformers `AutoTokenizer` loading works with `trust_remote_code=False`
+- Traditional Chinese and Bopomofo smoke roundtrip works
+- No tokens matching `<|audio_[0-9]+|>`
+- No dense timestamp token ranges matching `<|ts_[0-9]+|>`,
+  `<|timestamp_[0-9]+|>`, or `<|time_[0-9]+|>`

config.json

@@ -0,0 +1,6 @@
+{
+  "model_type": "gpt2",
+  "tokenizer_class": "GPT2TokenizerFast",
+  "vocab_size": 114688,
+  "model_max_length": 131072
+}

evaluation_report.json

@@ -0,0 +1,58 @@
+{
+  "ok": true,
+  "failures": [],
+  "sections": {
+    "Timestamp / rich transcription checks": {
+      "rich_transcription_token_ids": {
+        "<|transcript_start|>": 114670,
+        "<|transcript_end|>": 114671,
+        "<|segment_start|>": 114672,
+        "<|segment_end|>": 114673,
+        "<|speaker|>": 114674,
+        "<|start_time|>": 114675,
+        "<|end_time|>": 114676,
+        "<|duration|>": 114677,
+        "<|content|>": 114678,
+        "<|non_speech_event|>": 114679
+      },
+      "timestamp_precision_digits": 2,
+      "json_roundtrip_ok": true,
+      "dense_timestamp_tokens_found": [],
+      "missing_rich_transcription_tokens": [],
+      "timestamp_strings_present": {
+        "0.00": true,
+        "3.42": true,
+        "10.25": true,
+        "3575.50": true
+      },
+      "non_speech_labels_present": {
+        "[Silence]": true,
+        "[Noise]": true,
+        "[Music]": true,
+        "[Unintelligible Speech]": true
+      },
+      "text_roundtrip_ok": {
+        "traditional_chinese": true,
+        "bopomofo_mixed_romanization": true,
+        "json_syntax": true
+      },
+      "required_fields_ok": true,
+      "parse_error": null
+    }
+  },
+  "rich_transcription_token_ids": {
+    "<|transcript_start|>": 114670,
+    "<|transcript_end|>": 114671,
+    "<|segment_start|>": 114672,
+    "<|segment_end|>": 114673,
+    "<|speaker|>": 114674,
+    "<|start_time|>": 114675,
+    "<|end_time|>": 114676,
+    "<|duration|>": 114677,
+    "<|content|>": 114678,
+    "<|non_speech_event|>": 114679
+  },
+  "timestamp_precision_digits": 2,
+  "json_roundtrip_ok": true,
+  "dense_timestamp_tokens_found": []
+}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    "<|system|>",
+    "<|user_channel|>",
+    "<|assistant_channel|>",
+    "<|task:speech_to_text|>",
+    "<|task:text_to_speech|>",
+    "<|input_audio_start|>",
+    "<|input_audio_end|>",
+    "<|audio_ref_start|>",
+    "<|audio_ref_end|>",
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|speech_start|>",
+    "<|speech_end|>",
+    "<|transcript_start|>",
+    "<|transcript_end|>",
+    "<|segment_start|>",
+    "<|segment_end|>",
+    "<|speaker|>",
+    "<|start_time|>",
+    "<|end_time|>",
+    "<|duration|>",
+    "<|content|>",
+    "<|non_speech_event|>",
+    "<|retrieval_result_start|>",
+    "<|retrieval_result_end|>",
+    "<|ocr_start|>",
+    "<|ocr_end|>",
+    "<|image_start|>",
+    "<|image_end|>",
+    "<|video_start|>",
+    "<|video_end|>"
+  ],
+  "pad_token": "<|pad|>",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "unk_token": "<|unk|>"
+}

tokenizer_config.json

@@ -0,0 +1,100 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": [
+    "<|system|>",
+    "<|user_channel|>",
+    "<|assistant_channel|>",
+    "<|task:speech_to_text|>",
+    "<|task:text_to_speech|>",
+    "<|input_audio_start|>",
+    "<|input_audio_end|>",
+    "<|audio_ref_start|>",
+    "<|audio_ref_end|>",
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|speech_start|>",
+    "<|speech_end|>",
+    "<|transcript_start|>",
+    "<|transcript_end|>",
+    "<|segment_start|>",
+    "<|segment_end|>",
+    "<|speaker|>",
+    "<|start_time|>",
+    "<|end_time|>",
+    "<|duration|>",
+    "<|content|>",
+    "<|non_speech_event|>",
+    "<|retrieval_result_start|>",
+    "<|retrieval_result_end|>",
+    "<|ocr_start|>",
+    "<|ocr_end|>",
+    "<|image_start|>",
+    "<|image_end|>",
+    "<|video_start|>",
+    "<|video_end|>"
+  ],
+  "model_max_length": 131072,
+  "model_type": "byte_level_bpe",
+  "pad_token": "<|pad|>",
+  "rich_transcription": {
+    "allow_non_speech_events": true,
+    "compact_json": true,
+    "default_format": "json_segments",
+    "enabled": true,
+    "include_content": true,
+    "include_speaker": true,
+    "include_start_end": true,
+    "no_dense_timestamp_tokens": true,
+    "timestamp_precision_digits": 2,
+    "timestamp_unit": "seconds"
+  },
+  "special_tokens": [
+    "<|pad|>",
+    "<|bos|>",
+    "<|eos|>",
+    "<|unk|>",
+    "<|system|>",
+    "<|user_channel|>",
+    "<|assistant_channel|>",
+    "<|task:speech_to_text|>",
+    "<|task:text_to_speech|>",
+    "<|input_audio_start|>",
+    "<|input_audio_end|>",
+    "<|audio_ref_start|>",
+    "<|audio_ref_end|>",
+    "<|audio_start|>",
+    "<|audio_end|>",
+    "<|speech_start|>",
+    "<|speech_end|>",
+    "<|transcript_start|>",
+    "<|transcript_end|>",
+    "<|segment_start|>",
+    "<|segment_end|>",
+    "<|speaker|>",
+    "<|start_time|>",
+    "<|end_time|>",
+    "<|duration|>",
+    "<|content|>",
+    "<|non_speech_event|>",
+    "<|retrieval_result_start|>",
+    "<|retrieval_result_end|>",
+    "<|ocr_start|>",
+    "<|ocr_end|>",
+    "<|image_start|>",
+    "<|image_end|>",
+    "<|video_start|>",
+    "<|video_end|>"
+  ],
+  "strict_no_dense_timestamp_tokens": true,
+  "tokenizer_class": "GPT2TokenizerFast",
+  "unk_token": "<|unk|>",
+  "vocab_size": 114688,
+  "padding_side": "right",
+  "truncation_side": "right",
+  "clean_up_tokenization_spaces": false,
+  "no_audio_codec_tokens": true,
+  "no_dense_timestamp_tokens": true,
+  "fix_mistral_regex": true
+}