Change tokenizer v1.0

core/interviewer.py

@@ -8,22 +8,24 @@ Generates random MBTI-style questions using a fine-tuned model.
 """
 
 import random, torch, re
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import AutoModelForSeq2SeqLM, T5Tokenizer
 
 # --------------------------------------------------------------
 # 1️⃣ Настройки модели
 # --------------------------------------------------------------
 QG_MODEL = "f3nsmart/ft-flan-t5-base-qgen"
 
-tokenizer = AutoTokenizer.from_pretrained(QG_MODEL)
+# ❗ Используем "slow" SentencePiece токенайзер
+tokenizer = T5Tokenizer.from_pretrained(QG_MODEL, use_fast=False)
 model = AutoModelForSeq2SeqLM.from_pretrained(QG_MODEL)
+
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device).eval()
 
-print(f"✅ Loaded interviewer model: {QG_MODEL}")
+print(f"✅ Loaded interviewer model (slow tokenizer): {QG_MODEL}")
 
 # --------------------------------------------------------------
-# 2️⃣ Базовые промпты (легкий "seed", без инструкций)
+# 2️⃣ Базовые seed-промпты (без инструкций)
 # --------------------------------------------------------------
 PROMPTS = [
     "Personality and emotions.",