import tiktoken tokenizer = tiktoken.get_encoding("gpt2") #select the tokenizer vocabulary model to tokenize the given input def tokenize(text): token_ids = tokenizer.encode(text) #encode the given text input so that it will convert it to the token values tokens=[] for token_id in token_ids: tokens.append(tokenizer.decode([token_id])) #we use [token_id], ie [], because, decoder expects the input in form of a list, therefore we need to pass it as a list only token_count = len(token_ids) return token_ids,tokens,token_count