39f5261
1
2
3
4
5
6
7
from transformers import AutoTokenizer def load_tokenizer(model_name: str): print(f"🔤 Chargement du tokenizer pour {model_name}...") tokenizer = AutoTokenizer.from_pretrained(model_name) return tokenizer
