Load Vectara model via transformers pipeline, not CrossEncoder

CrossEncoder conflicts with HHEMv2's custom architecture on newer
transformers (tied weights resolution mismatch). Vectara's own docs
recommend transformers.pipeline("text-classification").

Pipeline output: {"label": "Factually Consistent", "score": float}
Normalise: if label is inconsistent, faithfulness = 1 - score.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Dockerfile

@@ -12,9 +12,10 @@ RUN pip install --no-cache-dir -r requirements.txt
 
 # Pre-download models so first request isn't slow on HF Spaces
 RUN python -c "\
-from sentence_transformers import SentenceTransformer, CrossEncoder; \
+from sentence_transformers import SentenceTransformer; \
+from transformers import pipeline; \
 SentenceTransformer('all-MiniLM-L6-v2'); \
-CrossEncoder('vectara/hallucination_evaluation_model', trust_remote_code=True)"
+pipeline('text-classification', model='vectara/hallucination_evaluation_model', trust_remote_code=True)"
 
 COPY knowledge/ ./knowledge/
 COPY backend/   ./backend/

backend/grader.py

@@ -5,16 +5,18 @@ Metrics:
   pii_leakage        — regex scan for PII patterns in response
   token_budget       — response within allowed token ceiling
   answer_relevancy   — cosine similarity between query and response embeddings
-  faithfulness       — NLI cross-encoder: is response grounded in retrieved context?
+  faithfulness       — Vectara HHEM v2: RAG faithfulness probability per (doc, response) pair
   chain_terminology  — deterministic: client-specific terms used (via RosettaStone)
 """
 
 import re
 import logging
 from dataclasses import dataclass, field
+from typing import Any
 
-from sentence_transformers import SentenceTransformer, CrossEncoder
+from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
+from transformers import pipeline as hf_pipeline
 
 from config import EMBEDDER_MODEL
 from rosetta import check_terminology
@@ -22,7 +24,7 @@ from rosetta import check_terminology
 log = logging.getLogger(__name__)
 
 _embedder: SentenceTransformer | None = None
-_nli_model: CrossEncoder | None = None
+_nli_model: Any = None
 
 NLI_MODEL = "vectara/hallucination_evaluation_model"
 
@@ -35,11 +37,17 @@ def get_embedder() -> SentenceTransformer:
     return _embedder
 
 
-def get_nli_model() -> CrossEncoder:
-    """Return the shared NLI cross-encoder, loading it on first call."""
+def get_nli_model() -> Any:
+    """Return the shared Vectara faithfulness pipeline, loading it on first call."""
     global _nli_model
     if _nli_model is None:
-        _nli_model = CrossEncoder(NLI_MODEL, trust_remote_code=True)
+        _nli_model = hf_pipeline(
+            "text-classification",
+            model=NLI_MODEL,
+            trust_remote_code=True,
+            truncation=True,
+            max_length=512,
+        )
     return _nli_model
 
 
@@ -153,9 +161,13 @@ def grade_faithfulness(response: str, context: str) -> GradeResult:
     if not raw_chunks:
         return GradeResult(metric="faithfulness", passed=False, score=0.0, detail="No context")
     chunks = [_strip_chunk_title(c) for c in raw_chunks]
-    # Vectara model: predict((document, response)) → faithfulness probability in [0, 1]
-    pairs = [(chunk, response) for chunk in chunks]
-    scores = model.predict(pairs)
+    # Vectara HHEM v2: text-classification pipeline, label "Factually Consistent" = faithful
+    pairs = [{"text": chunk, "text_pair": response} for chunk in chunks]
+    results = model(pairs)
+    scores = [
+        r["score"] if "consistent" in r["label"].lower() else 1.0 - r["score"]
+        for r in results
+    ]
     score = float(max(scores))
     passed = score >= FAITHFULNESS_THRESHOLD
     return GradeResult(