core/tokenizer.py

from transformers import AutoTokenizer

def load_tokenizer(model_name: str):
    print(f"🔤 Chargement du tokenizer pour {model_name}...")
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    return tokenizer