app/utils/token_counter.py

from app.utils.config import config_manager

config = config_manager.get_config()


class InputTooLargeError(ValueError):
    def __init__(self, message="Input too large"):
        super().__init__(message)


def count_tokens(messages: list[dict]) -> int:

    total_chars = sum(len(msg["content"]) for msg in messages)
    token_count = total_chars // 4

    if token_count > config.nlp["max_input_tokens"]:
        raise InputTooLargeError()

    return token_count