Adding OpenAI models aand llm-as-judge options

.gitignore

@@ -9,4 +9,4 @@ data/
 !src/data/
 .venv/
 .env
-
+.idea

app/app.py

@@ -77,6 +77,7 @@ EVAL_METHOD_CHOICES = [
     ("Local NLI (free, no API key needed)", "local_nli"),
     ("Groq Judge – LLaMA-70B (needs GROQ_API_KEY)", "groq"),
     ("OpenAI Judge – GPT-4o-mini (needs OPENAI_API_KEY)", "openai"),
+    ("OpenAI GPT-4o Judge – highest quality (needs OPENAI_API_KEY)", "openai_gpt4"),
 ]
 
 
@@ -290,23 +291,26 @@ def _gold_comparison_df(pred: dict, gold: dict) -> pd.DataFrame:
 
 def run_single_query(dataset, embedding_model, chunk_label, retrieval_strategy, query_rewrite,
                      n_variants, reranker_choice, llm_model, dense_k, bm25_k, fusion_top_k, top_n,
-                     question, trace_on, reference, gold_scores, eval_method):
+                     question, trace_on, reference, gold_scores, eval_method, use_threshold):
     if not question or not question.strip():
         raise gr.Error("Please enter a question.")
 
     pipeline, _ = get_pipeline(dataset, embedding_model, chunk_label, reranker_choice, retrieval_strategy,
                                 query_rewrite, n_variants, llm_model, dense_k, bm25_k, fusion_top_k, top_n)
 
-    # Judge eval needs individual expanded doc texts, which live in trace data.
-    use_judge = eval_method in ("groq", "openai")
+    use_judge = eval_method in ("groq", "openai", "openai_gpt4")
     run_trace = bool(trace_on) or use_judge
+    use_judge_relevance = use_judge and not use_threshold
 
-    result = pipeline.answer(question, trace=run_trace)
+    result = pipeline.answer(question, trace=run_trace, use_threshold=bool(use_threshold))
 
     eval_trace = None
     if use_judge:
         try:
-            scores, eval_trace = score_result_with_judge_traced(result, question, backend=eval_method)
+            scores, eval_trace = score_result_with_judge_traced(
+                result, question, backend=eval_method,
+                use_judge_relevance=use_judge_relevance,
+            )
         except Exception as exc:
             raise gr.Error(f"Judge evaluation failed: {exc}")
     else:
@@ -411,15 +415,9 @@ def _sanitize_stats_for_json(stats_df: pd.DataFrame) -> list:
 
 # -- Benchmark tab ------------------------------------------------------------
 
-# Max concurrent LLM pipelines during a benchmark run.
-# Each slot may issue up to 2-3 API calls (query-transform + generation + judge).
-# 5 is safe for Groq's free tier (~30 RPM); increase for paid tiers.
-_BENCH_CONCURRENCY = 5
-
-
 async def run_benchmark(dataset, embedding_model, chunk_label, retrieval_strategy, query_rewrite,
                         n_variants, reranker_choice, llm_model, dense_k, bm25_k, fusion_top_k, top_n,
-                        n_samples, eval_method, progress=gr.Progress()):
+                        n_samples, eval_method, concurrency, use_threshold, progress=gr.Progress()):
     pipeline, entry = get_pipeline(dataset, embedding_model, chunk_label, reranker_choice, retrieval_strategy,
                                    query_rewrite, n_variants, llm_model, dense_k, bm25_k, fusion_top_k, top_n)
     import random as _rng
@@ -443,7 +441,8 @@ async def run_benchmark(dataset, embedding_model, chunk_label, retrieval_strateg
     cached = cache.get_cached_rows(results_df, chash)
     cached_idx = set(cached["row_index"].tolist())
 
-    use_judge = eval_method in ("groq", "openai")
+    use_judge = eval_method in ("groq", "openai", "openai_gpt4")
+    use_judge_relevance = use_judge and not use_threshold
 
     def _safe_float(series_row, col):
         val = series_row.get(col, float("nan"))
@@ -452,7 +451,7 @@ async def run_benchmark(dataset, embedding_model, chunk_label, retrieval_strateg
     def _fmt(v):
         return "N/A" if (v is None or (isinstance(v, float) and math.isnan(v))) else f"{v:.3f}"
 
-    sem = asyncio.Semaphore(_BENCH_CONCURRENCY)
+    sem = asyncio.Semaphore(int(concurrency))
     sample_list = list(sample.iterrows())
     total = len(sample_list)
     completed_count = 0
@@ -469,11 +468,14 @@ async def run_benchmark(dataset, embedding_model, chunk_label, retrieval_strateg
         reference = row.get("response", "") or None
 
         async with sem:
-            result = await pipeline.aanswer(question, trace=use_judge)
+            result = await pipeline.aanswer(question, trace=use_judge, use_threshold=bool(use_threshold))
 
             if use_judge:
                 try:
-                    scores = await score_result_with_judge_async(result, question, backend=eval_method)
+                    scores = await score_result_with_judge_async(
+                        result, question, backend=eval_method,
+                        use_judge_relevance=use_judge_relevance,
+                    )
                     log.info("[benchmark] row=%s  adh=%s  rel=%s  util=%s  comp=%s",
                              i, _fmt(scores['adherence']), _fmt(scores['relevance']),
                              _fmt(scores['utilization']), _fmt(scores['completeness']))
@@ -834,7 +836,7 @@ with gr.Blocks(title="Modular RAG Explorer", css=_CSS) as demo:
                 llm_dd = gr.Dropdown(
                     choices=_opt_choices("llm_models"),
                     value=RUNTIME_OPTIONS["llm_models"][0]["id"],
-                    label="LLM (Groq)",
+                    label="Generation LLM",
                 )
 
             # ── 5. Evaluation ─────────────────────────────────────────────
@@ -845,8 +847,19 @@ with gr.Blocks(title="Modular RAG Explorer", css=_CSS) as demo:
                     label="Method",
                     info=(
                         "Local NLI: free, on-device. "
-                        "Groq/OpenAI Judge: LLM-as-Judge "
-                        "(requires the corresponding API key as a Space secret)."
+                        "LLM Judge: uses the RAGBench TRACe prompt (section 7.4) "
+                        "to evaluate all four metrics in one call. "
+                        "Requires the corresponding API key as a Space secret."
+                    ),
+                )
+                threshold_cb = gr.Checkbox(
+                    label="Cross-encoder rejection gate",
+                    value=True,
+                    info=(
+                        "When checked: reject queries whose cross-encoder relevance "
+                        "falls below 0.3 and skip LLM generation (current behaviour). "
+                        "When unchecked (RAGBench mode): always generate — the judge "
+                        "computes relevance retrospectively from sentence keys."
                     ),
                 )
 
@@ -919,7 +932,15 @@ with gr.Blocks(title="Modular RAG Explorer", css=_CSS) as demo:
                 with gr.Tab("Benchmark"):
                     benchmark_state = gr.State(None)
 
-                    n_samples_dd = gr.Dropdown(choices=[5, 10, 25, 50, 100], value=10, label="Sample size")
+                    with gr.Row():
+                        n_samples_dd = gr.Dropdown(
+                            choices=[5, 10, 25, 50, 100], value=10, label="Sample size",
+                        )
+                        concurrency_dd = gr.Dropdown(
+                            choices=[1, 2, 3, 5, 10, 15, 20], value=5,
+                            label="Concurrency",
+                            info="Max parallel LLM calls. 5 is safe for Groq free tier (~30 RPM); raise for paid tiers.",
+                        )
                     run_bench_btn = gr.Button("Run Benchmark", variant="primary")
 
                     summary_df   = gr.Dataframe(label="Mean predicted vs gold scores per metric")
@@ -1057,7 +1078,7 @@ with gr.Blocks(title="Modular RAG Explorer", css=_CSS) as demo:
         inputs=[
             dataset_dd, embedding_dd, chunk_dd, retrieval_dd, rewrite_dd, n_variants_sl,
             reranker_dd, llm_dd, dense_k_sl, bm25_k_sl, fusion_top_k_sl, top_n_sl,
-            question_tb, trace_cb, reference_state, gold_scores_state, eval_method_dd,
+            question_tb, trace_cb, reference_state, gold_scores_state, eval_method_dd, threshold_cb,
         ],
         outputs=[
             answer_box,
@@ -1075,7 +1096,7 @@ with gr.Blocks(title="Modular RAG Explorer", css=_CSS) as demo:
         inputs=[
             dataset_dd, embedding_dd, chunk_dd, retrieval_dd, rewrite_dd, n_variants_sl,
             reranker_dd, llm_dd, dense_k_sl, bm25_k_sl, fusion_top_k_sl, top_n_sl,
-            n_samples_dd, eval_method_dd,
+            n_samples_dd, eval_method_dd, concurrency_dd, threshold_cb,
         ],
         outputs=[summary_df, per_q_df, stats_df_out, bench_bar, benchmark_state],
     )

app/scoring.py

@@ -76,13 +76,20 @@ async def score_result_with_judge_async(
     result: dict,
     question: str,
     backend: str,
+    use_judge_relevance: bool = False,
 ) -> dict:
-    """Async version of score_result_with_judge() — use from the benchmark pipeline."""
-    relevance = result["relevance"]
+    """Async version of score_result_with_judge().
+
+    use_judge_relevance=True  (threshold checkbox unchecked / RAGBench mode):
+      relevance = fraction of doc sentences the judge marks as relevant (TRACe eq. 2).
+    use_judge_relevance=False (threshold checkbox checked / default):
+      relevance = cross-encoder sigmoid score (preserved from pipeline).
+    """
+    cross_encoder_relevance = result["relevance"]
 
     if result.get("rejected") or not result["context"].strip():
         return {
-            "relevance": relevance,
+            "relevance": cross_encoder_relevance,
             "adherence": math.nan,
             "completeness": math.nan,
             "utilization": math.nan,
@@ -93,7 +100,8 @@ async def score_result_with_judge_async(
 
     judge = get_llm_judge(backend)
     scores = await judge.ascore(question, docs_texts, result["answer"])
-    scores["relevance"] = relevance
+    if not use_judge_relevance:
+        scores["relevance"] = cross_encoder_relevance
     return scores
 
 
@@ -101,18 +109,18 @@ def score_result_with_judge(
     result: dict,
     question: str,
     backend: str,
+    use_judge_relevance: bool = False,
 ) -> dict:
-    """Score via LLM-as-Judge (Groq or OpenAI).
+    """Score via LLM-as-Judge (Groq, OpenAI, or GPT-4o).
 
-    Requires the result to have been produced with ``trace=True`` so that
-    the expanded doc texts are available in ``result['trace']['expansions']``.
-    The cross-encoder relevance from the pipeline is preserved.
+    use_judge_relevance=True  (RAGBench mode): relevance from judge sentence keys.
+    use_judge_relevance=False (default):       relevance from cross-encoder (preserved).
     """
-    relevance = result["relevance"]
+    cross_encoder_relevance = result["relevance"]
 
     if result.get("rejected") or not result["context"].strip():
         return {
-            "relevance": relevance,
+            "relevance": cross_encoder_relevance,
             "adherence": math.nan,
             "completeness": math.nan,
             "utilization": math.nan,
@@ -123,7 +131,8 @@ def score_result_with_judge(
 
     judge = get_llm_judge(backend)
     scores = judge.score(question, docs_texts, result["answer"])
-    scores["relevance"] = relevance  # keep cross-encoder relevance
+    if not use_judge_relevance:
+        scores["relevance"] = cross_encoder_relevance
     return scores
 
 
@@ -167,21 +176,27 @@ def score_result_traced(result: dict, reference: str = None) -> tuple[dict, dict
     return scores, trace
 
 
-def score_result_with_judge_traced(result: dict, question: str, backend: str) -> tuple[dict, dict]:
+def score_result_with_judge_traced(
+    result: dict,
+    question: str,
+    backend: str,
+    use_judge_relevance: bool = False,
+) -> tuple[dict, dict]:
     """score_result_with_judge() with full judge prompt/response returned as a second value."""
-    relevance = result["relevance"]
+    cross_encoder_relevance = result["relevance"]
 
     if result.get("rejected") or not result["context"].strip():
-        scores = {"relevance": relevance, "adherence": math.nan,
+        scores = {"relevance": cross_encoder_relevance, "adherence": math.nan,
                   "completeness": math.nan, "utilization": math.nan}
         return scores, {"method": backend, "rejected": True}
 
-    expansions  = result.get("trace", {}).get("expansions", [])
-    docs_texts  = [e["text"] for e in expansions] if expansions else [result["context"]]
+    expansions = result.get("trace", {}).get("expansions", [])
+    docs_texts = [e["text"] for e in expansions] if expansions else [result["context"]]
 
-    judge  = get_llm_judge(backend)
+    judge = get_llm_judge(backend)
     scores, judge_trace = judge.score_traced(question, docs_texts, result["answer"])
-    scores["relevance"] = relevance
+    if not use_judge_relevance:
+        scores["relevance"] = cross_encoder_relevance
     judge_trace["method"] = backend
     return scores, judge_trace
 

src/evaluation/llm_judge.py

@@ -1,23 +1,24 @@
 """LLM-as-Judge evaluation for RAG quality metrics.
 
-Replicates the sentence-keyed judge pipeline from the capstone notebook:
+Implements the TRACe evaluation framework from RAGBench (Friel et al., 2025):
 - Splits retrieved docs and the generated answer into sentence-keyed strings.
-- Sends a structured prompt to an LLM judge via LiteLLM with JSON mode so the
-  response is constrained to valid JSON.
-- Validates the response against a Pydantic schema — missing fields get safe
-  defaults instead of KeyError / NaN.
-- Derives adherence, relevance, utilization, and completeness on [0, 1].
-
-Supported backends (passed as model strings to LiteLLM):
-  "groq"      → groq/llama-3.3-70b-versatile
-  "openai"    → openai/gpt-4o-mini
-  Any string containing "/" is treated as a raw LiteLLM model string.
-
-Metric definitions (notebook-compatible):
-- adherence   : 1 if judge says the answer is overall supported, else 0.
-- relevance   : fraction of doc sentences the judge marks as relevant.
-- utilization : fraction of doc sentences the judge says were used.
-- completeness: fraction of relevant sentences that were actually utilized.
+- Sends a structured prompt to an LLM judge (section 7.4 of the paper) that asks
+  the model to identify relevant, utilized, and supported sentences.
+- Validates the response against a Pydantic schema — missing fields get safe defaults.
+- Derives adherence, relevance, utilization, and completeness on [0, 1] using the
+  TRACe equations (paper section 3.2).
+
+Supported backends:
+  "groq"        → groq/llama-3.3-70b-versatile
+  "openai"      → openai/gpt-4o-mini   (cost-efficient judge)
+  "openai_gpt4" → openai/gpt-4o        (highest quality, closest to paper)
+  Any "/" string is treated as a raw LiteLLM model string.
+
+Metric definitions (RAGBench-compatible):
+  relevance   : Σ Len(Rᵢ) / Σ Len(dᵢ)  — fraction of doc sentences that are relevant.
+  utilization : Σ Len(Uᵢ) / Σ Len(dᵢ)  — fraction of doc sentences that were used.
+  completeness: Len(R ∩ U) / Len(R)     — fraction of relevant sentences that were used.
+  adherence   : 1 if overall_supported else 0.
 """
 
 from typing import Literal
@@ -26,69 +27,121 @@ import numpy as np
 from litellm import acompletion, completion
 from pydantic import BaseModel, Field
 
-from src import settings
 from src.generation.llm_client import _sync_api_keys
 from src.reranking.reranker import split_sentences
 
 # ---------------------------------------------------------------------------
-# Pydantic schema for judge output
+# Pydantic schema — matches RAGBench section 7.4 JSON structure
 # ---------------------------------------------------------------------------
 
 class _SentenceSupport(BaseModel):
     response_sentence_key: str = ""
+    explanation: str = ""
     supporting_sentence_keys: list[str] = Field(default_factory=list)
     fully_supported: bool = False
 
 
 class JudgeOutput(BaseModel):
-    overall_supported: bool = False
+    relevance_explanation: str = ""
     all_relevant_sentence_keys: list[str] = Field(default_factory=list)
-    all_utilized_sentence_keys: list[str] = Field(default_factory=list)
+    overall_supported_explanation: str = ""
+    overall_supported: bool = False
     sentence_support_information: list[_SentenceSupport] = Field(default_factory=list)
+    all_utilized_sentence_keys: list[str] = Field(default_factory=list)
 
 
 # ---------------------------------------------------------------------------
-# Prompt
+# System prompt
 # ---------------------------------------------------------------------------
 
 _SYSTEM_PROMPT = (
     "You are a strict RAG evaluation judge. "
-    "Respond with a JSON object only. Use only sentence keys provided in the prompt."
+    "Respond with a valid JSON object only — no text before or after. "
+    "Use only the sentence keys provided in the prompt."
 )
 
+# ---------------------------------------------------------------------------
+# Evaluation prompt — aligned with RAGBench Appendix 7.4
+# ---------------------------------------------------------------------------
+
 _EVAL_PROMPT = """\
-Evaluate the RAG response using the provided documents. Return a JSON object.
+I asked someone to answer a question based on one or more documents.
+Your task is to review their response and assess whether or not each sentence \
+in that response is supported by text in the documents. And if so, which \
+sentences in the documents provide that support. You will also tell me which \
+of the documents contain useful information for answering the question, and \
+which of the documents the answer was sourced from.
+
+Here are the documents, each split into sentences. Alongside each sentence \
+is an associated key (e.g. '0_0', '0_1', '1_0') that you can use to refer to it:
 
-Documents (sentence-keyed):
+```
 {documents_text}
+```
 
-Question:
+The question was:
+```
 {question}
+```
 
-Response (sentence-keyed):
+Here is the response, split into sentences. Alongside each sentence is an \
+associated key (e.g. 'r_0', 'r_1') that you can use to refer to it. \
+Note that these keys are unique to the response and are not related to the document keys:
+
+```
 {answer_text}
+```
 
-JSON structure:
+You must respond with a JSON object matching this schema:
 
+```
 {{
-  "overall_supported": true,
-  "all_relevant_sentence_keys": ["0_0"],
-  "all_utilized_sentence_keys": ["0_0"],
+  "relevance_explanation": string,
+  "all_relevant_sentence_keys": [string],
+  "overall_supported_explanation": string,
+  "overall_supported": boolean,
   "sentence_support_information": [
     {{
-      "response_sentence_key": "r_0",
-      "supporting_sentence_keys": ["0_0"],
-      "fully_supported": true
+      "response_sentence_key": string,
+      "explanation": string,
+      "supporting_sentence_keys": [string],
+      "fully_supported": boolean
     }}
-  ]
+  ],
+  "all_utilized_sentence_keys": [string]
 }}
-
-Rules:
-- Document keys look like: 0_0, 0_1, 1_0, 1_1. Use ONLY keys present above.
-- Response keys look like: r_0, r_1, r_2.
-- all_relevant_sentence_keys: every document sentence useful for answering the question.
-- all_utilized_sentence_keys: document sentences the response actually drew from.
-- supporting_sentence_keys must be empty when a response sentence is unsupported.\
+```
+
+Field definitions:
+
+relevance_explanation: Step-by-step breakdown of which document sentences contain \
+useful information for answering the question and why.
+
+all_relevant_sentence_keys: List of all document sentence keys (e.g. '0_0') that \
+are relevant to the question. Include every sentence that is useful for answering, \
+even if it was not used in the response or only parts are useful. \
+Base this solely on the documents and question — ignore the response. \
+Omit sentences that would not impact someone's ability to answer the question.
+
+overall_supported_explanation: Step-by-step breakdown assessing whether the response \
+as a whole is supported by the documents. Assess each claim separately before \
+drawing an overall conclusion.
+
+overall_supported: Boolean — true if the response as a whole is supported by the \
+documents. Reflects the conclusion of overall_supported_explanation.
+
+sentence_support_information: One object per response sentence:
+  - response_sentence_key: the key used above (e.g. 'r_0').
+  - explanation: why the sentence is or is not supported.
+  - supporting_sentence_keys: document sentence keys that support this response \
+sentence. Must be empty if unsupported. Special values allowed: \
+"supported_without_sentence" (generally supported but no specific sentence), \
+"general" (general/transition statement), "well_known_fact", "numerical_reasoning".
+  - fully_supported: true only if the sentence is completely grounded in the documents.
+
+all_utilized_sentence_keys: Document sentence keys that the response drew from. \
+Include every sentence that either directly supported or was implicitly used to \
+construct the answer. Omit sentences that were not used.\
 """
 
 
@@ -111,7 +164,7 @@ def build_keyed_response(answer: str) -> dict[str, str]:
 
 
 def _make_eval_prompt(keyed_docs: dict, question: str, keyed_answer: dict) -> str:
-    docs_text = "\n".join(f"{k}. {v}" for k, v in keyed_docs.items())
+    docs_text   = "\n".join(f"{k}. {v}" for k, v in keyed_docs.items())
     answer_text = "\n".join(f"{k}. {v}" for k, v in keyed_answer.items())
     return _EVAL_PROMPT.format(
         documents_text=docs_text,
@@ -121,15 +174,21 @@ def _make_eval_prompt(keyed_docs: dict, question: str, keyed_answer: dict) -> st
 
 
 # ---------------------------------------------------------------------------
-# Metric computation
+# Metric computation — TRACe equations (paper section 3.2)
 # ---------------------------------------------------------------------------
 
 def compute_judge_scores(judge: JudgeOutput, keyed_docs: dict) -> dict:
-    """Derive [0,1] metric scores from a validated JudgeOutput."""
-    valid = set(keyed_docs.keys())
+    """Derive [0,1] TRACe metric scores from a validated JudgeOutput.
+
+    relevance   = |Rᵢ| / |dᵢ|  (RAGBench eq. 2)
+    utilization = |Uᵢ| / |dᵢ|  (RAGBench eq. 3)
+    completeness= |R ∩ U| / |R| (RAGBench eq. 4)
+    adherence   = overall_supported (boolean → float)
+    """
+    valid    = set(keyed_docs.keys())
     relevant = set(judge.all_relevant_sentence_keys) & valid
     utilized = set(judge.all_utilized_sentence_keys) & valid
-    total = len(valid)
+    total    = len(valid)
 
     relevance    = len(relevant) / total if total else 0.0
     utilization  = len(utilized) / total if total else 0.0
@@ -149,13 +208,17 @@ def compute_judge_scores(judge: JudgeOutput, keyed_docs: dict) -> dict:
 # ---------------------------------------------------------------------------
 
 _BACKEND_MODELS = {
-    "groq":   "groq/llama-3.3-70b-versatile",
-    "openai": "openai/gpt-4o-mini",
+    "groq":        "groq/llama-3.3-70b-versatile",
+    "openai":      "openai/gpt-4o-mini",
+    "openai_gpt4": "openai/gpt-4o",
 }
 
 _ZERO_SCORES = {"adherence": 0.0, "relevance": 0.0, "utilization": 0.0, "completeness": 0.0}
-_EMPTY_TRACE = {"prompt": "", "raw_response": "", "judge": None,
-                "relevant_keys": [], "utilized_keys": [], "valid_keys_count": 0}
+_EMPTY_TRACE = {
+    "prompt": "", "raw_response": "", "judge": None,
+    "relevant_keys": [], "utilized_keys": [], "valid_keys_count": 0,
+    "relevance_explanation": "", "overall_supported_explanation": "",
+}
 
 
 # ---------------------------------------------------------------------------
@@ -163,30 +226,29 @@ _EMPTY_TRACE = {"prompt": "", "raw_response": "", "judge": None,
 # ---------------------------------------------------------------------------
 
 class LlmJudge:
-    """LLM-as-Judge scorer backed by LiteLLM.
+    """LLM-as-Judge scorer using the RAGBench TRACe evaluation prompt.
 
-    Supports:
-    - ``"groq"``   : LLaMA-3.3-70B via Groq (GROQ_API_KEY).
-    - ``"openai"`` : GPT-4o-mini via OpenAI (OPENAI_API_KEY).
-    - Any raw LiteLLM model string (e.g. ``"openrouter/..."``).
+    Backends:
+      "groq"        — LLaMA-3.3-70B via Groq (GROQ_API_KEY)
+      "openai"      — GPT-4o-mini via OpenAI (OPENAI_API_KEY)
+      "openai_gpt4" — GPT-4o via OpenAI (OPENAI_API_KEY) — closest to paper
 
-    Both ``score`` / ``score_traced`` (sync) and ``ascore`` / ``ascore_traced``
-    (async) are provided.  Use the async variants from the benchmark pipeline.
+    Sync: score() / score_traced()
+    Async: ascore() / ascore_traced()  — use from benchmark pipeline
     """
 
     def __init__(
         self,
-        backend: Literal["groq", "openai"] | str = "groq",
+        backend: Literal["groq", "openai", "openai_gpt4"] | str = "groq",
         model: str | None = None,
     ):
         self.backend = backend
-        self.model = model or _BACKEND_MODELS.get(backend, backend)
+        self.model   = model or _BACKEND_MODELS.get(backend, backend)
         _sync_api_keys()
 
     # -- sync -----------------------------------------------------------------
 
     def _call_llm(self, prompt: str) -> str:
-        """Sync LLM call with JSON mode. Returns raw JSON string."""
         resp = completion(
             model=self.model,
             messages=[
@@ -194,53 +256,46 @@ class LlmJudge:
                 {"role": "user",   "content": prompt},
             ],
             temperature=0.0,
-            max_tokens=3000,
+            max_tokens=4096,
             response_format={"type": "json_object"},
             num_retries=3,
         )
         return resp.choices[0].message.content
 
     def score(self, question: str, docs_texts: list[str], answer: str) -> dict:
-        """Run judge and return metric scores dict."""
         keyed_docs   = build_sentence_keyed_docs(docs_texts)
         keyed_answer = build_keyed_response(answer)
-
         if not keyed_docs:
             return dict(_ZERO_SCORES)
-
-        prompt = _make_eval_prompt(keyed_docs, question, keyed_answer)
-        raw    = self._call_llm(prompt)
-        judge  = JudgeOutput.model_validate_json(raw)
+        raw   = self._call_llm(_make_eval_prompt(keyed_docs, question, keyed_answer))
+        judge = JudgeOutput.model_validate_json(raw)
         return compute_judge_scores(judge, keyed_docs)
 
     def score_traced(self, question: str, docs_texts: list[str], answer: str) -> tuple[dict, dict]:
-        """Same as score() but also returns a trace dict."""
         keyed_docs   = build_sentence_keyed_docs(docs_texts)
         keyed_answer = build_keyed_response(answer)
-
         if not keyed_docs:
             return dict(_ZERO_SCORES), dict(_EMPTY_TRACE)
-
         prompt = _make_eval_prompt(keyed_docs, question, keyed_answer)
         raw    = self._call_llm(prompt)
         judge  = JudgeOutput.model_validate_json(raw)
         scores = compute_judge_scores(judge, keyed_docs)
-
-        valid = set(keyed_docs.keys())
-        trace = {
-            "prompt":           prompt,
-            "raw_response":     raw,
-            "judge":            judge.model_dump(),
-            "relevant_keys":    sorted(set(judge.all_relevant_sentence_keys) & valid),
-            "utilized_keys":    sorted(set(judge.all_utilized_sentence_keys) & valid),
-            "valid_keys_count": len(valid),
+        valid  = set(keyed_docs.keys())
+        trace  = {
+            "prompt":                        prompt,
+            "raw_response":                  raw,
+            "judge":                         judge.model_dump(),
+            "relevant_keys":                 sorted(set(judge.all_relevant_sentence_keys) & valid),
+            "utilized_keys":                 sorted(set(judge.all_utilized_sentence_keys) & valid),
+            "valid_keys_count":              len(valid),
+            "relevance_explanation":         judge.relevance_explanation,
+            "overall_supported_explanation": judge.overall_supported_explanation,
         }
         return scores, trace
 
     # -- async ----------------------------------------------------------------
 
     async def _acall_llm(self, prompt: str) -> str:
-        """Async LLM call with JSON mode. Returns raw JSON string."""
         resp = await acompletion(
             model=self.model,
             messages=[
@@ -248,45 +303,39 @@ class LlmJudge:
                 {"role": "user",   "content": prompt},
             ],
             temperature=0.0,
-            max_tokens=3000,
+            max_tokens=4096,
             response_format={"type": "json_object"},
             num_retries=3,
         )
         return resp.choices[0].message.content
 
     async def ascore(self, question: str, docs_texts: list[str], answer: str) -> dict:
-        """Async version of score() — use from the benchmark pipeline."""
         keyed_docs   = build_sentence_keyed_docs(docs_texts)
         keyed_answer = build_keyed_response(answer)
-
         if not keyed_docs:
             return dict(_ZERO_SCORES)
-
-        prompt = _make_eval_prompt(keyed_docs, question, keyed_answer)
-        raw    = await self._acall_llm(prompt)
-        judge  = JudgeOutput.model_validate_json(raw)
+        raw   = await self._acall_llm(_make_eval_prompt(keyed_docs, question, keyed_answer))
+        judge = JudgeOutput.model_validate_json(raw)
         return compute_judge_scores(judge, keyed_docs)
 
     async def ascore_traced(self, question: str, docs_texts: list[str], answer: str) -> tuple[dict, dict]:
-        """Async version of score_traced()."""
         keyed_docs   = build_sentence_keyed_docs(docs_texts)
         keyed_answer = build_keyed_response(answer)
-
         if not keyed_docs:
             return dict(_ZERO_SCORES), dict(_EMPTY_TRACE)
-
         prompt = _make_eval_prompt(keyed_docs, question, keyed_answer)
         raw    = await self._acall_llm(prompt)
         judge  = JudgeOutput.model_validate_json(raw)
         scores = compute_judge_scores(judge, keyed_docs)
-
-        valid = set(keyed_docs.keys())
-        trace = {
-            "prompt":           prompt,
-            "raw_response":     raw,
-            "judge":            judge.model_dump(),
-            "relevant_keys":    sorted(set(judge.all_relevant_sentence_keys) & valid),
-            "utilized_keys":    sorted(set(judge.all_utilized_sentence_keys) & valid),
-            "valid_keys_count": len(valid),
+        valid  = set(keyed_docs.keys())
+        trace  = {
+            "prompt":                        prompt,
+            "raw_response":                  raw,
+            "judge":                         judge.model_dump(),
+            "relevant_keys":                 sorted(set(judge.all_relevant_sentence_keys) & valid),
+            "utilized_keys":                 sorted(set(judge.all_utilized_sentence_keys) & valid),
+            "valid_keys_count":              len(valid),
+            "relevance_explanation":         judge.relevance_explanation,
+            "overall_supported_explanation": judge.overall_supported_explanation,
         }
         return scores, trace

src/generation/llm_client.py

@@ -22,24 +22,22 @@ litellm.num_retries = 3
 
 DEFAULT_MODEL = "groq/llama-3.1-8b-instant"
 
+# Matches RAGBench section 7.3 / 7.7 "LONG" generation prompt.
 NEGATIVE_RESPONSE = (
-    "I don't have enough information in the provided context to answer this question."
+    "The documents are missing some of the information required to answer the question."
 )
 
 PROMPT_TEMPLATE = """\
-You are a helpful assistant answering questions using ONLY the provided context.
+You are a chatbot providing answers to user queries. You will be given one or more context documents. \
+Use the information in the documents to answer the question.
 
-Context:
-{context}
-
-Question: {question}
+If the documents do not provide enough information for you to answer the question, then say \
+"{negative_response}" Don't quote things not in the documents. Don't try to make up an answer.
 
-Instructions:
-- Answer using only information from the context above.
-- If the context does not contain enough information to answer, respond exactly with: "{negative_response}"
-- Be concise and direct.
+Context Documents:
+{context}
 
-Answer:"""
+Question: {question}"""
 
 
 def _resolve_model(model: str) -> str:

src/pipeline/rag_pipeline.py

@@ -172,13 +172,20 @@ class RagPipeline:
 
         return {"context": context, "reranked": reranked, "relevance": relevance}
 
-    def answer(self, question: str, trace: bool = False) -> dict:
+    def answer(self, question: str, trace: bool = False, use_threshold: bool = True) -> dict:
+        """Run the full pipeline synchronously.
+
+        use_threshold=True  (default): reject queries whose cross-encoder
+          relevance score falls below cfg.relevance_threshold (current behaviour).
+        use_threshold=False (RAGBench mode): always generate — the LLM judge
+          evaluates relevance retrospectively from sentence keys.
+        """
         cfg = self.config
         trace_dict = {} if trace else None
 
         retrieval = self.assemble_context(question, trace=trace_dict)
 
-        if retrieval["relevance"] < cfg.relevance_threshold:
+        if use_threshold and retrieval["relevance"] < cfg.relevance_threshold:
             result = {**retrieval, "answer": NEGATIVE_RESPONSE, "rejected": True}
             if trace_dict is not None:
                 trace_dict["prompt"] = None
@@ -200,19 +207,22 @@ class RagPipeline:
             result["trace"] = trace_dict
         return result
 
-    async def aanswer(self, question: str, trace: bool = False) -> dict:
+    async def aanswer(self, question: str, trace: bool = False, use_threshold: bool = True) -> dict:
         """Async variant of answer() for concurrent benchmark execution.
 
         CPU-bound retrieval and reranking run in a thread pool so they don't
         block the asyncio event loop.  The LLM generation call is truly async
         via litellm.acompletion.
+
+        use_threshold=False skips the cross-encoder rejection gate so the LLM
+        judge can evaluate relevance retrospectively (RAGBench approach).
         """
         cfg = self.config
         trace_dict = {} if trace else None
 
         retrieval = await asyncio.to_thread(self.assemble_context, question, trace_dict)
 
-        if retrieval["relevance"] < cfg.relevance_threshold:
+        if use_threshold and retrieval["relevance"] < cfg.relevance_threshold:
             result = {**retrieval, "answer": NEGATIVE_RESPONSE, "rejected": True}
             if trace_dict is not None:
                 trace_dict["prompt"] = None