File size: 277 Bytes
01f199c | 1 2 3 4 5 6 7 8 9 10 11 12 13 | # pip3 install transformers
# python3 deepseek_tokenizer.py
import transformers
chat_tokenizer_dir = "./"
tokenizer = transformers.AutoTokenizer.from_pretrained(
chat_tokenizer_dir, trust_remote_code=True
)
result = tokenizer.encode("Hello!")
print(result)
|