remove old tokenizer

claude_tokenizer.py

@@ -1,112 +0,0 @@
-import re
-import json
-from typing import List, Dict
-
-class ClaudeTokenizer:
-    def __init__(self, config_file: str, algorithm: str = "trie"):
-        with open(config_file, "r") as f:
-            config = json.load(f)
-
-        self.vocab = config["vocab"]
-        self.vocab_size = config["n_vocab_size"]
-        self.pat_str = config["pat_str"]
-        self.special_tokens = config["special_tokens"]
-
-        self.token_to_id = {token: i for i, token in enumerate(self.vocab)}
-        self.id_to_token = {i: token for token, i in self.token_to_id.items()}
-
-        for token, id in self.special_tokens.items():
-            self.token_to_id[token] = id
-            self.id_to_token[id] = token
-
-        self.pat = re.compile(self.pat_str)
-        self.vocab_trie = self._build_trie(self.vocab)
-
-        self.algorithm = algorithm
-        if algorithm not in ["trie", "linear"]:
-            raise ValueError("Invalid algorithm. Choose 'trie' or 'linear'.")
-
-    def _build_trie(self, vocab: List[str]) -> Dict:
-        trie = {}
-        for token in vocab:
-            current = trie
-            for char in token:
-                if isinstance(current, str):
-                    break
-                if char not in current:
-                    current[char] = {}
-                current = current[char]
-            if isinstance(current, dict):
-                current["*"] = token
-        return trie
-
-    def tokenize(self, text: str) -> List[str]:
-        if self.algorithm == "trie":
-            tokens = []
-            for part in self.pat.findall(text):
-                tokens.extend(self._tokenize_part_trie(part))
-            return tokens
-        else:
-            return self._tokenize_part_linear(text)
-
-    def encode(self, text: str) -> List[int]:
-        tokens = self.tokenize(text)
-        return [
-            self.token_to_id.get(token, self.special_tokens["<META>"])
-            for token in tokens
-        ]
-
-    def decode(self, ids: List[int]) -> str:
-        return "".join(self.id_to_token.get(id, "") for id in ids)
-
-    def _tokenize_part_trie(self, text: str) -> List[str]:
-        tokens = []
-        while text:
-            current = self.vocab_trie
-            longest_match = ""
-            for i, char in enumerate(text):
-                if char not in current:
-                    break
-                current = current[char]
-                if "*" in current:
-                    longest_match = current["*"]
-            if longest_match:
-                tokens.append(longest_match)
-                text = text[len(longest_match):]
-            else:
-                tokens.append(text[0])
-                text = text[1:]
-        return tokens
-
-    def _tokenize_part_linear(self, text: str) -> List[str]:
-        tokens = []
-        while text:
-            longest_match = ""
-            for token in self.vocab:
-                if text.startswith(token) and len(token) > len(longest_match):
-                    longest_match = token
-            if longest_match:
-                tokens.append(longest_match)
-                text = text[len(longest_match):]
-            else:
-                tokens.append(text[0])
-                text = text[1:]
-        return tokens
-
-
-# Usage example
-if __name__ == "__main__":
-    # Choose the algorithm: "trie" or "linear"
-    algorithm = "linear"  # or "trie"
-    
-    tokenizer = ClaudeTokenizer("tokenizer_config.json", algorithm=algorithm)
-
-    test_text = """Hello! It's nice to meet you. How can I assist you today? I'm here to help with any questions you might have or tasks you need help with."""
-    tokens = tokenizer.tokenize(test_text)
-    print(f"Tokens ({algorithm}):", tokens)
-
-    encoded = tokenizer.encode(test_text)
-    print(f"Encoded ({algorithm}):", encoded)
-
-    decoded = tokenizer.decode(encoded)
-    print(f"Decoded ({algorithm}):", decoded)