44c5827
1
2
3
4
5
6
7
8
9
import tiktoken try: tokenizer = tiktoken.get_encoding("cl100k_base") except: tokenizer = tiktoken.get_encoding("gpt2") # get the number of tokens in a text def count_tokens(text: str) -> int: return len(tokenizer.encode(text))
