Upload tokenizer.py

tokenizer.py

@@ -0,0 +1,23 @@
+class DigitTokenizer:
+    def __init__(self):
+        self.digit_char = "0123456789*%="
+        self.special_tokens = ["<bos>", "<pad>", "<eos>", "<unk>"]
+        self.vocab_list = list(self.digit_char) + self.special_tokens
+        
+        self.char_to_int = {char: i for i, char in enumerate(self.vocab_list)}
+        self.int_to_char = {i: char for i, char in enumerate(self.vocab_list)}
+        self.vocab_size = len(self.vocab_list)
+
+    def encode(self, input_string: str) -> list[int]:
+        return [self.char_to_int.get(char, self.char_to_int["<unk>"]) for char in input_string]
+
+    def decode(self, token_ids: list[int]) -> str:
+        decoded_chars = []
+        for token_id in token_ids:
+            char = self.int_to_char.get(token_id)
+            if char == "<eos>":
+                break
+            if char is not None and char not in ["<pad>", "<unk>"]:
+                decoded_chars.append(char)
+                
+        return "".join(decoded_chars)