Pushing the changes for judge as LLM

backend/engines/BM25Enginer.py

@@ -0,0 +1,82 @@
+import math
+from typing import Any, Dict, List
+
+from backend.routers.retrieval_router import get_keywords
+
+
+class BM25Engine:
+    def __init__(
+        self, documents: List[str], doc_ids: List[str], metadatas: List[Dict[str, Any]]
+    ):
+        self.doc_ids = doc_ids
+        self.documents = documents
+        self.metadatas = metadatas
+        self.corpus_size = len(documents)
+        self.k1 = 1.5
+        self.b = 0.75
+
+        self.tokenized_docs = [get_keywords(doc) for doc in documents]
+        self.doc_length = [len(doc) for doc in self.tokenized_docs]
+        self.avg_doc_len = sum(self.doc_length) / max(1, self.corpus_size)
+
+        self.doc_tfs = []
+        self.dfs = {}
+        self.build_frequencies()
+
+    def build_frequencies(self):
+        for doc in self.tokenized_docs:
+            tfs = {}
+            for term in doc:
+                tfs[term] = tfs.get(term, 0) + 1
+            self.doc_tfs.append(tfs)
+            for term in doc:
+                self.dfs[term] = self.dfs.get(term, 0) + 1
+
+    def idf(self, term):
+        df = self.dfs.get(term, 0)
+        return math.log(1 + (self.corpus_size - df + 0.5) / (df + 0.5))
+
+    def score(self, query: str, doc_index: int) -> float:
+        query_terms = get_keywords(query)
+
+        total_score = 0.0
+        doc_len = self.doc_length[doc_index]
+        tfs = self.doc_tfs[doc_index]
+
+        for term in query_terms:
+            tf = tfs.get(term, 0)
+            if tf > 0:
+                term_idf = self.idf(term)
+
+                # BM25 term weighting formula
+                numerator = tf * (self.k1 + 1)
+                denominator = tf + self.k1 * (
+                    1 - self.b + self.b * (doc_len / self.avg_doc_len)
+                )
+
+                total_score += term_idf * (numerator / denominator)
+
+        return total_score
+
+    def search(self, query: str, top_k: int = 3) -> list:
+        scores = []
+        # Score all docs
+        for idx in range(self.corpus_size):
+            s = self.score(query, idx)
+            scores.append((s, idx))
+
+        # Sort descending by score
+        scores.sort(key=lambda x: x[0], reverse=True)
+
+        # Return documents with structured metadata
+        results = []
+        for score, idx in scores[:top_k]:
+            results.append(
+                {
+                    "id": self.doc_ids[idx],
+                    "text": self.documents[idx],
+                    "metadata": self.metadatas[idx],
+                    "score": score,
+                }
+            )
+        return results

backend/models/schemas.py

@@ -151,6 +151,12 @@ class ChunkScore(BaseModel):
             / 4,
             2,
         )
+        
+        # Log the discrepancy if the LLM's math is wrong
+        if abs(self.overall - expected) > 0.01:
+            print(f"\n\033[93m[JUDGE RAW] overall: {self.overall} (dims: query_relevance={self.query_relevance}, answer_completeness={self.answer_completeness}, factual_plausibility={self.factual_plausibility}, clarity={self.clarity})\033[0m")
+            print(f"\033[91m[VALIDATOR] expected overall: {expected} → overriding LLM's claimed {self.overall}\033[0m\n")
+            
         self.overall = expected
         return self
 

backend/routers/retrieval_router.py

@@ -3,7 +3,7 @@ from backend.engines import llm_client
 from backend.engines.llm_client import OllamaClient
 from backend.constants import system_instructions
 import asyncio
-from typing import Any, Optional
+from typing import Any, List, Optional
 from fastapi import APIRouter
 import json
 import re
@@ -16,6 +16,7 @@ from backend.models.schemas import (
     JudgeRequest,
     QueryRequest,
     QueryResponse,
+    RetrievalMode,
     RetrievedChunk,
 )
 from backend.storage.vector_store import VectorStore
@@ -48,6 +49,7 @@ async def retrieve(request: QueryRequest):
         request.strategy,
         request.top_k,
         request.search_text,
+        request.retrieval_mode,
     )
 
     query_coords = [0.0, 0.0]
@@ -116,21 +118,33 @@ async def judge(request: JudgeRequest):
     llm_client = OllamaClient()
     result = await llm_client.generate(prompt=prompt)
 
-    print(f"res:res: {result}")
-
     result_dict = json.loads(result)
 
     return JudgeResponse(**result_dict)
 
 
 async def process_retrieval(
-    model, search_text, strategy, top_k, original_query: Optional[str] = None
+    model,
+    search_text,
+    strategy,
+    top_k,
+    retrieval_mode,
+    original_query: Optional[str] = None,
 ):
     vector_store = VectorStore()
     embedding_engine = EmbeddingEngine(model.value)
     embeddings = await embedding_engine.generate_embeddings([search_text])
+    collection_name = f"{model.value}_{strategy.value}".replace(":", "-")
+    retrieval_response: Any
+    if retrieval_mode == RetrievalMode.DENSE:
+        """"""
+    elif retrieval_mode == RetrievalMode.SPARSE:
+        """"""
+    elif retrieval_mode == RetrievalMode.HYBRID:
+        """"""
+
     result: Any = await vector_store.retrieve(
-        collection_name=f"{model.value}_{strategy.value}".replace(":", "-"),
+        collection_name=collection_name,
         embeddings=embeddings,
         n_results=top_k,
     )
@@ -189,3 +203,15 @@ async def get_hyde_text(search_text):
     llm_client = OllamaClient()
     hyde_prompt.format(search_text=search_text)
     return await llm_client.generate(hyde_prompt, response_format=None)
+
+
+def rrf(dense_ranks: List[str], sparse_ranks: List[str], k: int = 60) -> List[tuple]:
+    rrf_scores = {}
+
+    for rank, doc_id in enumerate(dense_ranks):
+        rrf_scores[doc_id] = rrf_scores.get(doc_id, 0.0) + 1.0 / (k + rank + 1)
+
+    for rank, doc_id in enumerate(sparse_ranks):
+        rrf_scores[doc_id] = rrf_scores.get(doc_id, 0.0) + 1.0 / (k + rank + 1)
+
+    return sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)

backend/storage/vector_store.py

@@ -29,5 +29,11 @@ class VectorStore:
         collection = self.get_collection(collection_name)
 
         return await asyncio.to_thread(
-            collection.query, query_embeddings=embeddings, n_results=n_results
+            collection.query,
+            query_embeddings=embeddings,
+            n_results=n_results,
         )
+
+    async def get_all_documents(self, collection_name: str):
+        collection = self.get_collection(collection_name)
+        return await asyncio.to_thread(collection.get)

dev.bat

@@ -1 +1 @@
-uv run uvicorn backend.main:app --reload --port 8080
+uv run uvicorn backend.main:app --reload --port 8000

frontend/app.js

@@ -72,6 +72,7 @@ const dom = {
   // Query Simulator elements
   queryInput: $("#query-input"),
   btnQuery: $("#btn-query"),
+  queryRetrievalMode: $("#query-retrieval-mode"),
   queryResultsDrawer: $("#query-results-drawer"),
   queryResultsList: $("#query-results-list"),
   closeDrawer: $("#close-drawer"),
@@ -1201,6 +1202,7 @@ async function runQuerySimulator() {
         embedding_model: state.config.embedding_model,
         strategy: state.strategy,
         top_k: 3,
+        retrieval_mode: dom.queryRetrievalMode ? dom.queryRetrievalMode.value : "dense",
       }),
     });
 
@@ -1340,6 +1342,7 @@ const domArena = {
   modelB: document.getElementById("arena-model-b"),
   strategyB: document.getElementById("arena-strategy-b"),
   resultsB: document.getElementById("arena-results-b"),
+  retrievalMode: document.getElementById("arena-retrieval-mode"),
 };
 
 if (domArena.btnOpen) {
@@ -1389,6 +1392,7 @@ if (domArena.btnFight) {
           strategy_a: domArena.strategyA.value,
           model_b: domArena.modelB.value,
           strategy_b: domArena.strategyB.value,
+          retrieval_mode: domArena.retrievalMode ? domArena.retrievalMode.value : "dense",
         }),
       });
 

frontend/index.html

@@ -243,7 +243,12 @@
                 <button id="btn-open-arena" class="btn-sm" style="background: var(--superman-red); color: white; border: none; padding: 4px 12px; border-radius: 4px; cursor: pointer; font-weight: bold; font-family: var(--font-sans);">⚔️ Arena Comparison</button>
               </div>
               <div class="query-input-wrapper">
-                <input type="text" id="query-input" placeholder="Type a concept query to fetch semantically similar chunks (e.g. 'linear regression')..." />
+                <select id="query-retrieval-mode" class="select-input" style="width: 110px; flex-shrink: 0; display: none;">
+                  <option value="dense" selected>Dense</option>
+                  <option value="sparse">Sparse</option>
+                  <option value="hybrid">Hybrid</option>
+                </select>
+                <input type="text" id="query-input" placeholder="Type a concept query to fetch chunks (e.g. 'linear regression')..." />
                 <button id="btn-query" class="btn-query">🔍 Query</button>
               </div>
               <div class="query-results-drawer" id="query-results-drawer" style="display: none;">
@@ -305,6 +310,11 @@
           </div>
           
           <div class="arena-query-bar">
+            <select id="arena-retrieval-mode" class="select-input" style="width: 110px; flex-shrink: 0; display: none;">
+              <option value="dense" selected>Dense</option>
+              <option value="sparse">Sparse</option>
+              <option value="hybrid">Hybrid</option>
+            </select>
             <input type="text" id="arena-query-input" placeholder="Type a query to compare configurations (e.g., 'What is Clark Kent's weakness?')..." />
             <button class="btn-run" id="btn-arena-fight" style="width: 150px; flex-shrink: 0;">🔥 FIGHT!</button>
           </div>

frontend/styles.css

@@ -698,6 +698,25 @@ input[type="range"]::-moz-range-thumb {
 /* ============================================================
    RETRIEVAL X-RAY HIGHLIGHTS (TASK 3.4)
    ============================================================ */
+/* --- Select Dropdown Styling --- */
+.select-input {
+  padding: 8px 12px;
+  background-color: var(--bg-input);
+  border: 1px solid var(--border-default);
+  border-radius: var(--radius-md);
+  color: var(--text-primary);
+  font-family: var(--font-sans);
+  font-size: 0.8rem;
+  outline: none;
+  transition: all var(--transition-base);
+  cursor: pointer;
+}
+
+.select-input:focus {
+  border-color: var(--superman-blue);
+  box-shadow: 0 0 0 2px var(--superman-blue-muted);
+}
+
 .xray-content.search-active .chunk-highlight {
   opacity: 0.3;
   transition: all 0.3s ease;