token_engine.py

# token_engine.py

try:
    import tiktoken
except ImportError:
    tiktoken = None

class TokenEngine:
    def __init__(self, model_name: str = "gpt-3.5-turbo"):
        if tiktoken:
            try:
                self.encoder = tiktoken.encoding_for_model(model_name)
            except Exception:
                self.encoder = tiktoken.get_encoding("cl100k_base")
        else:
            self.encoder = None

    def count_tokens(self, text: str) -> int:
        if self.encoder:
            return len(self.encoder.encode(text))
        return len(text.split())