File size: 560 Bytes
f56d742
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
import tiktoken

tokenizer = tiktoken.get_encoding("gpt2") #select the tokenizer vocabulary model to tokenize the given input

def tokenize(text):
    token_ids = tokenizer.encode(text) #encode the given text input so that it will convert it to the token values

    tokens=[]

    for token_id in token_ids:
        tokens.append(tokenizer.decode([token_id])) #we use [token_id], ie [], because, decoder expects the input in form of a list, therefore we need to pass it as a list only

    token_count = len(token_ids)
    return token_ids,tokens,token_count