Umap fix

app.py

@@ -1,13 +1,3 @@
-import os
-import sys
-
-DEBUG = False  # ← set False to hide ALL noise
-
-os.environ["TRANSFORMERS_VERBOSITY"] = "error"
-os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
-os.environ["TOKENIZERS_PARALLELISM"] = "false"
-os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
-
 import time
 import numpy as np
 from flask import Flask, render_template, request, jsonify
@@ -20,67 +10,64 @@ from src.retrieval.query import (
 )
 from src.generation.generate import generate_answer, build_prompt, build_context
 
-class SuppressOutput:
-    def __enter__(self):
-        if not DEBUG:
-            self._stdout = sys.stdout
-            self._stderr = sys.stderr
-            sys.stdout = open(os.devnull, "w")
-            sys.stderr = open(os.devnull, "w")
-
-    def __exit__(self, *args):
-        if not DEBUG:
-            sys.stdout.close()
-            sys.stderr.close()
-            sys.stdout = self._stdout
-            sys.stderr = self._stderr
+import os
+import logging
+import warnings
+
+# ---------------- ENV + LOGGING ----------------
+os.environ["TRANSFORMERS_VERBOSITY"] = "error"
+os.environ["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+warnings.filterwarnings("ignore")
 
+logging.getLogger("transformers").setLevel(logging.ERROR)
+logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
+logging.getLogger("urllib3").setLevel(logging.ERROR)
+
+# ---------------- APP ----------------
 app = Flask(__name__)
 
-with SuppressOutput():
-    _tokenizer = AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5")
+# ---------------- TOKENIZER ----------------
+_tokenizer = AutoTokenizer.from_pretrained("BAAI/bge-small-en-v1.5")
 
 
-# -----------------------------
-# BM25 IDF
-# -----------------------------
+# ---------------- IDF ----------------
 def _build_idf(bm25):
     return {term: max(0.0, float(val)) for term, val in bm25.idf.items()}
 
+
 _idf_map = _build_idf(bm25_index)
 _max_idf = max(_idf_map.values()) if _idf_map else 1.0
 
 
-# -----------------------------
-# PCA FIT
-# -----------------------------
+# ---------------- PCA ----------------
 def _fit_pca(n_components=128):
     import random
     from sentence_transformers import SentenceTransformer
 
     sample = random.sample(docs_all, min(200, len(docs_all)))
-
-    with SuppressOutput():
-        model = SentenceTransformer("BAAI/bge-small-en-v1.5")
-        embs = model.encode(sample, normalize_embeddings=True)
+    model = SentenceTransformer("BAAI/bge-small-en-v1.5")
+    embs = model.encode(sample, normalize_embeddings=True)
 
     n_comp = min(n_components, embs.shape[0], embs.shape[1])
     pca = PCA(n_components=n_comp)
     pca.fit(embs)
+
     return pca
 
 
 _pca = _fit_pca(128)
 
 
-# -----------------------------
-# ROUTES
-# -----------------------------
+# ---------------- ROUTES ----------------
+
 @app.route("/")
 def home():
     return render_template("index.html")
 
 
+# ---------------- QUERY ANALYSIS ----------------
 @app.route("/analyze_query", methods=["POST"])
 def analyze_query():
     data = request.json
@@ -100,22 +87,28 @@ def analyze_query():
     for tok in tokens:
         word = tok["token"].lstrip("##").lower()
         raw_idf = _idf_map.get(word, 0.0)
+
         tok["idf"] = round(raw_idf, 4)
         tok["idf_normalized"] = round(raw_idf / _max_idf, 4) if _max_idf else 0.0
 
     idf_vals = [t["idf"] for t in tokens]
     avg_idf = round(sum(idf_vals) / len(idf_vals), 4) if idf_vals else 0.0
     unique_toks = len({t["token"] for t in tokens})
-    complexity = round(min(1.0, (len(tokens) / 20) * 0.4 + (avg_idf / _max_idf) * 0.6), 3)
+
+    complexity = round(
+        min(1.0, (len(tokens) / 20) * 0.4 + (avg_idf / _max_idf) * 0.6),
+        3
+    )
 
     q_emb = embed_query(query)
     projected = _pca.transform(q_emb.reshape(1, -1))[0]
+
     p_min, p_max = projected.min(), projected.max()
 
-    normed = (
-        ((projected - p_min) / (p_max - p_min) * 2 - 1).tolist()
-        if p_max != p_min else [0.0] * len(projected)
-    )
+    if p_max != p_min:
+        normed = ((projected - p_min) / (p_max - p_min) * 2 - 1).tolist()
+    else:
+        normed = [0.0] * len(projected)
 
     return jsonify({
         "tokens": tokens,
@@ -129,6 +122,7 @@ def analyze_query():
     })
 
 
+# ---------------- MMR RERUN ----------------
 @app.route("/mmr_rerun", methods=["POST"])
 def mmr_rerun():
     if _session["query_emb"] is None:
@@ -155,18 +149,20 @@ def mmr_rerun():
     })
 
 
+# ---------------- MAIN RAG ----------------
 @app.route("/ask", methods=["POST"])
 def ask():
     data = request.json
     query = data.get("query")
 
-    if DEBUG:
-        print(f"\n[API QUERY]: {query}\n")
+    print(f"\n[API QUERY]: {query}\n")
 
+    # -------- RETRIEVE --------
     t0 = time.perf_counter()
     results, debug = retrieve(query)
     t_retrieve = time.perf_counter() - t0
 
+    # -------- SORT --------
     results = sorted(results, key=lambda x: (
         x["meta"].get("chunk_id", 0),
         x["meta"].get("global_chunk_id", 0)
@@ -176,22 +172,67 @@ def ask():
     metas = [r["meta"] for r in results]
     raw_scores = [float(r["rerank_score"]) for r in results]
 
+    # -------- CONTEXT --------
     context = build_context(docs, metas, raw_scores)
 
+    # -------- LLM --------
     t1 = time.perf_counter()
     prompt = build_prompt(query, context)
     answer = generate_answer(prompt)
     t_llm = time.perf_counter() - t1
 
+    # -------- SOURCES --------
     sources = [
-        {"title": meta.get("title", "Source"), "url": meta.get("url", "")}
+        {
+            "title": meta.get("title", "Source"),
+            "url": meta.get("url", "")
+        }
         for meta in metas
     ]
 
+    # -------- TIMINGS --------
     stage_timings = debug.get("timings", {})
     stage_timings["llm"] = round(t_llm * 1000)
     stage_timings["total"] = round((t_retrieve + t_llm) * 1000)
 
+    # -------- COMPARISON --------
+    score_lookup = debug.get("score_lookup", {})
+    full_rerank = debug.get("rerank_full", [])
+
+    hybrid_order = {
+        int(k): rank for rank, k in enumerate(score_lookup.keys())
+    }
+
+    comparison_rows = []
+
+    for post_rank, (idx, rerank_score) in enumerate(full_rerank):
+        idx = int(idx)
+        sk = score_lookup.get(str(idx), [0, 0, 0])
+
+        pre_rank = hybrid_order.get(idx, post_rank)
+
+        comparison_rows.append({
+            "idx": idx,
+            "pre_rank": pre_rank,
+            "post_rank": post_rank,
+            "rank_delta": pre_rank - post_rank,
+            "vector_score": round(float(sk[0]), 4),
+            "bm25_score": round(float(sk[1]), 4),
+            "hybrid_score": round(float(sk[2]), 4),
+            "rerank_score": round(float(rerank_score), 4),
+            "passed_threshold": float(rerank_score) >= 0.3,
+            "text_preview": " ".join(
+                docs_all[idx]
+                .replace("passage: ", "")
+                .strip()
+                .lstrip("`")
+                .split()
+            )[:120],
+            "text_full": docs_all[idx].replace("passage: ", ""),
+            "title": metas_all[idx].get("title", ""),
+        })
+
+    # -------- RESPONSE --------
     return jsonify({
         "answer": answer,
         "sources": sources,
@@ -199,12 +240,22 @@ def ask():
         "scores": raw_scores,
         "raw_scores": raw_scores,
         "debug": debug,
-        "timings": stage_timings
+        "timings": stage_timings,
+        "comparison_rows": comparison_rows,
+
+        # 🔥 CRITICAL (DO NOT CHANGE)
+        "mmr_data": {
+            "umap_coords": debug.get("umap_coords"),
+            "sim_matrix": debug.get("sim_matrix"),
+            "doc_indices": debug.get("doc_indices", []),
+            "sims": debug.get("sims", []),
+            "doc_previews": debug.get("doc_previews", []),
+            "mmr_selected": debug.get("mmr_selected", []),
+            "no_mmr_selected": debug.get("no_mmr_selected", []),
+        }
     })
 
 
-# -----------------------------
-# RUN
-# -----------------------------
+# ---------------- RUN ----------------
 if __name__ == "__main__":
-    app.run(debug=DEBUG)
+    app.run(debug=True)