app/utils/tokens.py

import tiktoken

def estimate_token_count(prompt: str, model: str) -> int:
    """
    Estimate the token count for a given prompt and model.

    Args:
        prompt (str): The input prompt to tokenize.
        model (str): The name of the model to use for token encoding.

    Returns:
        int: The estimated token count.
    """
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(prompt))