Spaces:
Sleeping
Sleeping
File size: 560 Bytes
f56d742 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | import tiktoken
tokenizer = tiktoken.get_encoding("gpt2") #select the tokenizer vocabulary model to tokenize the given input
def tokenize(text):
token_ids = tokenizer.encode(text) #encode the given text input so that it will convert it to the token values
tokens=[]
for token_id in token_ids:
tokens.append(tokenizer.decode([token_id])) #we use [token_id], ie [], because, decoder expects the input in form of a list, therefore we need to pass it as a list only
token_count = len(token_ids)
return token_ids,tokens,token_count
|