Create phi_tokenizer.py

phi_tokenizer.py

@@ -0,0 +1,23 @@
+class Tokenizer:
+   def __init__(self):
+       self.tokens = []
+
+   def tokenize(self, text):
+       for char in text:
+           self.tokens.append(ord(char))
+       return self.tokens
+
+   def detokenize(self, tokens):
+       text = ''.join(chr(token) for token in tokens)
+       return text
+
+def main():
+   phi_tokenizer = Tokenizer()
+   text = "hello world"
+   tokenized = phi_tokenizer.tokenize(text)
+   print(tokenized)
+   detokenized = phi_tokenizer.detokenize(tokenized)
+   print(detokenized)
+
+if __name__ == "__main__":
+   main()