Add Balochi SentencePiece BPE tokenizer

README.md

@@ -0,0 +1,40 @@
+---
+language:
+- bal
+license: apache-2.0
+tags:
+- balochi
+- tokenizer
+- sentencepiece
+- bpe
+- perso-arabic
+- low-resource-nlp
+---
+
+# Balochi SentencePiece BPE Tokenizer
+
+SentencePiece BPE tokenizer trained on the Mozilla Balochi Dataset.
+Designed for Balochi written in Perso-Arabic script.
+
+## Character Inventory
+- 38 standard Balochi letters (exact Unicode codepoints)
+- 6 atomic special sequences: ۓ یے ئے ءُ ءِ ءَ
+- Meaningful diacritics preserved: Fatha (ـَ) Damma (ـُ) Kasra (ـِ) Shadda (ـّ)
+- byte_fallback=True — zero [UNK] tokens guaranteed
+
+## Usage
+```python
+from transformers import AutoTokenizer
+tok = AutoTokenizer.from_pretrained("YOUR_USERNAME/balochi-tokenizer")
+tokens = tok.tokenize("دبستانانی وانشتءَ پیسر")
+print(tokens)
+```
+
+## Special Tokens
+| Token | ID |
+|---|---|
+| [PAD] | 0 |
+| [UNK] | 1 |
+| [CLS] | 2 |
+| [SEP] | 3 |
+| [MASK] | 4 |

tokenizer.json

@@ -0,0 +1,209 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "Sequence",
+    "normalizers": [
+      {
+        "type": "Strip",
+        "strip_left": false,
+        "strip_right": true
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": " {2,}"
+        },
+        "content": "▁"
+      }
+    ]
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "WhitespaceSplit"
+      },
+      {
+        "type": "Metaspace",
+        "replacement": "▁",
+        "prepend_scheme": "always",
+        "split": true
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "</s>",
+          "type_id": 0
+        }
+      }
+    ],
+    "special_tokens": {
+      "</s>": {
+        "id": "</s>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "</s>"
+        ]
+      },
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "always",
+    "split": true
+  },
+  "model": {
+    "type": "Unigram",
+    "unk_id": 3,
+    "vocab": [
+      [
+        "<s>",
+        0.0
+      ],
+      [
+        "[PAD]",
+        0.0
+      ],
+      [
+        "</s>",
+        0.0
+      ],
+      [
+        "[UNK]",
+        0.0
+      ],
+      [
+        "[MASK]",
+        0.0
+      ]
+    ],
+    "byte_fallback": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "cls_token": "[CLS]",
+  "eos_token": "</s>",
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {
+    "enable_sampling": false
+  },
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "[UNK]",
+  "language": "bal",
+  "keep_accents": true,
+  "strip_accents": false,
+  "do_lower_case": false
+}