Fix ML model loading: add sentencepiece + use_fast=False for XLMRoberta tokenizer

The tokenizer was failing with 'Converting from Tiktoken failed' because
sentencepiece package was missing from requirements. Also added use_fast=False
as a safety fallback for the slow tokenizer path.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

backend/app/api/health.py

@@ -78,7 +78,7 @@ async def model_diagnostics():
     try:
         from transformers import AutoModelForSequenceClassification, AutoTokenizer
         t0 = time.time()
-        tok = AutoTokenizer.from_pretrained(settings.sentiment_model, cache_dir=settings.model_cache_dir)
+        tok = AutoTokenizer.from_pretrained(settings.sentiment_model, cache_dir=settings.model_cache_dir, use_fast=False)
         model = AutoModelForSequenceClassification.from_pretrained(settings.sentiment_model, cache_dir=settings.model_cache_dir)
         model.eval()
         elapsed = round(time.time() - t0, 2)

backend/app/services/sentiment.py

@@ -42,6 +42,7 @@ def _load_model():
         _tokenizer = AutoTokenizer.from_pretrained(
             model_name,
             cache_dir=settings.model_cache_dir,
+            use_fast=False,
         )
         logger.info("tokenizer_loaded", model=model_name, elapsed=round(time.time() - t0, 2))
 

backend/requirements.txt

@@ -52,3 +52,5 @@ httpx==0.28.1
 # Utilities
 python-dotenv==1.0.1
 tenacity==9.0.0
+sentencepiece==0.2.0
+protobuf==5.29.3