tokenization.py

from model_registry import validate_model_id
import model_service

def count_tokens(model_id: str, text: str) -> int:
    """
    Get actual token count using the model's tokenizer.
    If the model is not loaded, it loads the tokenizer-only temporarily.
    """
    validate_model_id(model_id)
    
    # Check if the active tokenizer is the correct one
    if model_service.active_model_id == model_id and model_service.active_tokenizer is not None:
        tokenizer = model_service.active_tokenizer
    else:
        # Load tokenizer dynamically
        tokenizer = model_service.load_tokenizer_only(model_id)
        
    tokens = tokenizer.encode(text, add_special_tokens=False)
    return len(tokens)