tokenizer-visualizer / tokenizer.py
Pawan19012006's picture
Upload tokenizer.py with huggingface_hub
f56d742 verified
Raw History Blame Contribute Delete
560 Bytes
import tiktoken
tokenizer = tiktoken.get_encoding("gpt2") #select the tokenizer vocabulary model to tokenize the given input
def tokenize(text):
token_ids = tokenizer.encode(text) #encode the given text input so that it will convert it to the token values
tokens=[]
for token_id in token_ids:
tokens.append(tokenizer.decode([token_id])) #we use [token_id], ie [], because, decoder expects the input in form of a list, therefore we need to pass it as a list only
token_count = len(token_ids)
return token_ids,tokens,token_count