demo.py

from crayon import CrayonVocab

tokenizer = CrayonVocab(device="auto")

print("--- Testing Standard ---")
tokenizer.load_profile("standard")
tokens_std = tokenizer.tokenize("that is a test for the standard profile and lite profile and god")
print(tokens_std)
print(tokenizer.decode(tokens_std))

print("--- Testing Lite ---")
tokenizer.load_profile("lite")
tokens_lite = tokenizer.tokenize("my daughter")
print(tokens_lite)
print(tokenizer.decode(tokens_lite))