llm/tokenizer_utils.py

from __future__ import annotations

from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import BaseMessage
from langchain_core.prompts import ChatPromptTemplate


def get_llm_tokenizer(llm: BaseChatModel):
    inner = getattr(llm, "llm", None)
    pipeline = getattr(inner, "pipeline", None)
    if pipeline is not None:
        return getattr(pipeline, "tokenizer", None)
    return None


def count_text_tokens(tokenizer, text: str) -> int:
    if not text or tokenizer is None:
        return 0
    encoded = tokenizer.encode(text, add_special_tokens=False)
    return len(encoded)


def count_prompt_tokens(
    llm: BaseChatModel,
    prompt: ChatPromptTemplate,
    *,
    model_input: str,
    history: list[BaseMessage],
) -> int:
    tokenizer = get_llm_tokenizer(llm)
    if tokenizer is None:
        return 0
    messages = prompt.format_messages(input=model_input, history=history)
    parts: list[str] = []
    for message in messages:
        role = getattr(message, "type", message.__class__.__name__)
        parts.append(f"{role}: {message.content}")
    return count_text_tokens(tokenizer, "\n".join(parts))