Upload folder using huggingface_hub

README.md

@@ -0,0 +1,21 @@
+# Indic Tokenizer v2
+
+Custom SentencePiece Unigram tokenizer trained on:
+- Hindi, Tamil, Telugu corpora
+- Code-mixed Hinglish data
+
+## Features
+- 40–70% fewer tokens vs GPT-2
+- Script-aware tokenization
+- Better handling of Indic languages
+
+## Usage
+
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained(
+    "your-username/indic-tokenizer-v2",
+    trust_remote_code=True
+)
+
+print(tokenizer.tokenize("नमस्ते मित्र, कैसे हो?"))

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "pad_token": "<pad>"}

tokenizer.py

@@ -0,0 +1,27 @@
+import sentencepiece as spm
+from transformers import PreTrainedTokenizer
+
+class IndicTokenizer(PreTrainedTokenizer):
+    def __init__(self, vocab_file, **kwargs):
+        self.sp_model = spm.SentencePieceProcessor()
+        self.sp_model.load(vocab_file)
+        super().__init__(**kwargs)
+
+    def _tokenize(self, text):
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        return self.sp_model.id_to_piece(index)
+
+    def get_vocab(self):
+        return {self.sp_model.id_to_piece(i): i for i in range(self.sp_model.get_piece_size())}
+
+    def __len__(self):
+        return self.sp_model.get_piece_size()
+
+    @property
+    def vocab_size(self):
+        return self.sp_model.get_piece_size()