File size: 203 Bytes
708f4a3 | 1 2 3 4 5 6 7 8 9 10 | from crayon import CrayonVocab
vocab = CrayonVocab(device="auto")
vocab.load_profile("lite")
text = "Hello, world!"
tokens = vocab.tokenize(text)
print(tokens)
decode=vocab.decode(tokens)
print(decode) |