src/utils/utilities.py

import tiktoken

def count_num_tokens(text: str, model: str) -> int:
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))