package

.gitignore

@@ -0,0 +1 @@
+__pycache__/

README.md

@@ -11,7 +11,7 @@ language:
 The Mana Tokenizer is a custom-trained BPE tokenizer designed for Persian text. It is trained on a combination of huge Persian corpus. The tokenizer is built using the BPE with high character coverage to handle diverse Persian text.
 
 ## Quick Start
-
+You can encode/decode your data using Mana Tokenizer like this:
 ```python
 from mana_tokenizer import ManaTokenizer
 tokenizer = ManaTokenizer()
@@ -19,11 +19,17 @@ text = "سلام من یک متن تست برای تست این تست هستم.
 print(tokenizer.encode(text))
 print(tokenizer.decode(tokenizer.encode(text)))
 ```
+output should be:
+```
+[216, 179, 217, 132, 216, 167, 217, 133, 32, 217, 133, 217, 134, 32, 219, 140, 218, 169, 32, 217, 133, 216, 170, 217, 134, 32, 216, 170, 216, 179, 216, 170, 32, 216, 168, 216, 177, 216, 167, 219, 140, 32, 216, 170, 216, 179, 216, 170, 32, 216, 167, 219, 140, 217, 134, 32, 216, 170, 216, 179, 216, 170, 32, 217, 135, 216, 179, 216, 170, 217, 133, 46]
+سلام من یک متن تست برای تست این تست هستم.
+```
 
-You can also add special tokens
+You can also add special tokens:
 ```python
 tokenizer.register_special_tokens({"</s>": 100269})
 ```
+
 Batch encode:
 ```python
 tokenizer.batch_encode(["یک متن طولانی"])

__init__.py

@@ -1,3 +1,3 @@
 from .base import Tokenizer
 from .mana_tokenizer import ManaTokenizer
-import helper 
+from .helper import _process_string_scalar, render_token, merge

base.py

@@ -9,7 +9,7 @@ import os
 import regex as re
 import csv
 import time
-import helper
+from mana_tokenizer.helper import _process_string_scalar, render_token, merge
 
 class Tokenizer:
     """Base class for Tokenizers"""
@@ -97,7 +97,7 @@ class Tokenizer:
                 batch_size = len(item) // (self._cpus*2) or 1
                 batches = [item[i:i + batch_size] for i in range(0, len(item), batch_size)]
                 print(f'Processing {len(batches)} batches of size {batch_size}')
-                results = Parallel(n_jobs=self._cpus)(delayed(helper._process_string_scalar)(batch, self.compiled_pattern) for batch in batches)
+                results = Parallel(n_jobs=self._cpus)(delayed(_process_string_scalar)(batch, self.compiled_pattern) for batch in batches)
                 for result in results:  # Aggregate results into one Counter
                     ids.update(result)
             elif isinstance(item, IterableDataset):
@@ -171,12 +171,12 @@ class Tokenizer:
         inverted_merges = {idx: pair for pair, idx in self.merges.items()}
         with open(vocab_file, "w", encoding="utf-8") as f:  # Ensure this is also utf-8
             for idx, token in self.vocab.items():
-                s = helper.render_token(token)
+                s = render_token(token)
                 # find the children of this token, if any
                 if idx in inverted_merges:
                     idx0, idx1 = inverted_merges[idx]
-                    s0 = helper.render_token(self.vocab[idx0])
-                    s1 = helper.render_token(self.vocab[idx1])
+                    s0 = render_token(self.vocab[idx0])
+                    s1 = render_token(self.vocab[idx1])
                     f.write(f"[{s0}][{s1}] -> [{s}] {idx}\n")
                 else:
                     f.write(f"[{s}] {idx}\n")
@@ -237,7 +237,7 @@ class Tokenizer:
                 break   # nothing else can be merged
             # otherwise let's merge the best pair (lowest merge index)
             idx = self.merges[pair]
-            len_chunk = helper.merge(chunk, pair, idx, len_chunk)
+            len_chunk = merge(chunk, pair, idx, len_chunk)
         return chunk   # list of ints
 
     def encode_ordinary(self, text):

mana_tokenizer.py

@@ -1,4 +1,5 @@
-from .base import Tokenizer, get_stats, merge_batch_get_stats
+from .base import Tokenizer
+from .helper import get_stats, merge_batch_get_stats
 from heapq import nlargest
 import time
 

test.py

@@ -0,0 +1,5 @@
+from .mana_tokenizer import ManaTokenizer
+tokenizer = ManaTokenizer()
+text = "سلام من یک متن تست برای تست این تست هستم."
+print(tokenizer.encode(text))
+print(tokenizer.decode(tokenizer.encode(text)))