File size: 290 Bytes
a433a25 |
1 2 3 4 5 6 7 8 9 |
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(".")
print(tok.tokenize("Hello world! <user> write code </s>"))
text = "Hello world! <user> write code </s>"
ids = tok.encode(text)
print(ids)
print(tok.decode(ids))
print(tok.decode(ids, skip_special_tokens=True)) |