Deploy snapshot to HF (binaries stripped)

.github/workflows/push_to_hf_space.yml

@@ -0,0 +1,62 @@
+name: Sync to Hugging Face Space (always strip binaries)
+
+on:
+  push:
+    branches: [ "hf-deploy" ]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: hf-deploy         
+
+      - name: Create clean snapshot (remove .pkl/.faiss no matter what)
+        run: |
+          set -euxo pipefail
+
+          # Export current commit as plain files (no .git history)
+          rm -rf /tmp/snapshot
+          mkdir -p /tmp/snapshot
+          git archive --format=tar HEAD | tar -x -C /tmp/snapshot
+
+          echo "== Files matching .pkl/.faiss BEFORE removal =="
+          (cd /tmp/snapshot && find . -type f \( -name "*.pkl" -o -name "*.faiss" \) -print) || true
+
+          # Always delete these binaries (even if someone committed them)
+          (cd /tmp/snapshot && find . -type f \( -name "*.pkl" -o -name "*.faiss" \) -print -delete) || true
+
+          echo "== Files matching .pkl/.faiss AFTER removal =="
+          (cd /tmp/snapshot && find . -type f \( -name "*.pkl" -o -name "*.faiss" \) -print) || true
+
+          # Ensure they won't be committed into the snapshot repo
+          printf "\n# Never deploy vectorstore binaries to Spaces\n*.pkl\n*.faiss\n" >> /tmp/snapshot/.gitignore
+
+          # Build a new single-commit git repo from the cleaned snapshot
+          cd /tmp/snapshot
+          git init
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add -A
+          git commit -m "Deploy snapshot to HF (binaries stripped)"
+
+      - name: Push snapshot to Hugging Face Space (HF main)
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+          HF_USERNAME: ritup3          # CHANGE if needed
+          SPACE_NAME: PersonaRag       # CHANGE if needed
+        run: |
+          set -euxo pipefail
+          cd /tmp/snapshot
+
+          # Authenticated remote for HF Space
+          git remote add space https://$HF_USERNAME:$HF_TOKEN@huggingface.co/spaces/$HF_USERNAME/$SPACE_NAME
+
+          # Spaces deploy from branch "main"
+          git branch -M main
+          git push space main --force

.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+*.faiss
+*.pkl
+.env
+.DS_Store
+# Never deploy vectorstore binaries to Spaces
+*.pkl
+*.faiss

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -0,0 +1,11 @@
+---
+title: PersonaRag
+emoji: "🤖"
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "5.49.1"
+python_version: "3.10.13"
+app_file: app.py
+pinned: false
+---

app.py

@@ -0,0 +1,414 @@
+# app.py
+import os
+import json
+import logging
+from pathlib import Path
+from dotenv import load_dotenv
+import gradio as gr
+import time
+import threading
+import atexit
+
+from rag_core.index_builder import load_vectorstore
+from rag_core.rag_chain_helper import rewrite_question_with_history
+from rag_core.rag_chain import build_rag_chain
+from rag_core.evaluator import evaluate_answer
+from rag_core.index_builder import build_and_save_index
+
+from rag_core.config import VECTORSTORE_PATH
+
+# ---------- Refresh config ----------
+REFRESH_ENABLED = os.getenv("REFRESH_ENABLED", "true").lower() == "true"
+REFRESH_INTERVAL_SECONDS = int(os.getenv("REFRESH_INTERVAL_SECONDS", str(24 * 60 * 60)))
+REFRESH_AT_HOUR = int(os.getenv("REFRESH_AT_HOUR", "3"))
+REFRESH_AT_MINUTE = int(os.getenv("REFRESH_AT_MINUTE", "0"))
+REFRESH_ONLY_FIXED_URLS = os.getenv("REFRESH_ONLY_FIXED_URLS", "false").lower() == "true"
+
+state_lock = threading.RLock()
+stop_refresh_event = threading.Event()
+
+
+load_dotenv()
+
+# ---------- Logging (Model Flow) ----------
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+
+logger = logging.getLogger("model_flow")
+logger.setLevel(LOG_LEVEL)
+
+if not logger.handlers:
+    h = logging.StreamHandler()
+    h.setLevel(LOG_LEVEL)
+    formatter = logging.Formatter(
+        "%(asctime)s | %(levelname)s | %(name)s | %(message)s"
+    )
+    h.setFormatter(formatter)
+    logger.addHandler(h)
+
+def log_event(event: str, **payload):
+    """Structured-ish logging for tracing model flow."""
+    safe = {}
+    for k, v in payload.items():
+        try:
+            json.dumps(v)  # ensure serializable
+            safe[k] = v
+        except TypeError:
+            safe[k] = str(v)
+    logger.info("%s | %s", event, json.dumps(safe, ensure_ascii=False))
+
+
+# ---------- Global state ----------
+vectorstore = None
+rag_chain = None
+retriever = None
+system_prompt = None
+
+
+def init_rag():
+    global vectorstore, rag_chain, retriever, system_prompt
+
+    # HARD DISABLE: no crawling / no auto-index build
+    index_path = Path(VECTORSTORE_PATH) / "index.faiss"
+    if not index_path.exists():
+        n_chunks, _ = build_and_save_index()
+        log_event("refresh.index_built", mode="crawl", chunks=n_chunks)
+
+    vectorstore = load_vectorstore()
+
+    rag_chain, retriever, system_prompt = build_rag_chain(
+        vectorstore,
+        k=5,
+        max_docs=2
+    )
+    log_event("init_rag.ready", vectorstore_path=VECTORSTORE_PATH)
+
+
+init_rag()
+
+def refresh_rag_once():
+    """
+    Refetch website docs and rebuild the index + chain.
+    Never crashes the app; logs errors.
+    """
+    global vectorstore, rag_chain, retriever, system_prompt
+
+    log_event("refresh.start", only_fixed_urls=REFRESH_ONLY_FIXED_URLS)
+
+    try:
+        from rag_core.rag_chain import build_rag_chain
+        from rag_core.index_builder import load_vectorstore
+
+        n_chunks, _ = build_and_save_index()
+        log_event("refresh.index_built", mode="crawl", chunks=n_chunks)
+
+        # Reload from disk (ensures consistent serialization)
+        vs = load_vectorstore()
+
+        # Build new chain
+        new_chain, new_retriever, new_system_prompt = build_rag_chain(
+            vs,
+            k=5,
+            max_docs=2,
+        )
+
+        # Atomic swap
+        with state_lock:
+            vectorstore = vs
+            rag_chain = new_chain
+            retriever = new_retriever
+            system_prompt = new_system_prompt
+
+        log_event("refresh.done", status="ok")
+
+    except Exception as e:
+        log_event("refresh.error", error=str(e))
+
+def _seconds_until_next_run(hour: int, minute: int) -> int:
+    # compute sleep until next local time (hour:minute)
+    now = time.localtime()
+    target = time.mktime((
+        now.tm_year, now.tm_mon, now.tm_mday,
+        hour, minute, 0,
+        now.tm_wday, now.tm_yday, now.tm_isdst
+    ))
+    now_ts = time.time()
+    if target <= now_ts:
+        target += 24 * 60 * 60
+    return int(target - now_ts)
+
+def _daily_refresh_loop():
+    # small startup delay
+    time.sleep(3)
+
+    while not stop_refresh_event.is_set():
+        # sleep until next scheduled time
+        sleep_s = _seconds_until_next_run(REFRESH_AT_HOUR, REFRESH_AT_MINUTE)
+        log_event("refresh.sleep", seconds=sleep_s, at_hour=REFRESH_AT_HOUR, at_minute=REFRESH_AT_MINUTE)
+
+        # sleep in chunks so shutdown responds quickly
+        while sleep_s > 0 and not stop_refresh_event.is_set():
+            step = min(5, sleep_s)
+            time.sleep(step)
+            sleep_s -= step
+
+        if stop_refresh_event.is_set():
+            break
+
+        refresh_rag_once()
+
+def start_refresh_thread():
+    if not REFRESH_ENABLED:
+        log_event("refresh.disabled")
+        return
+    t = threading.Thread(target=_daily_refresh_loop, daemon=True)
+    t.start()
+    log_event("refresh.thread_started", daily_at=f"{REFRESH_AT_HOUR:02d}:{REFRESH_AT_MINUTE:02d}")
+
+atexit.register(lambda: stop_refresh_event.set())
+start_refresh_thread()
+
+
+
+# ---------- Helpers ----------
+def _history_to_text(history):
+    """Convert Gradio history ([[user, bot], ...]) to a readable text snippet."""
+    if not history:
+        return ""
+    lines = []
+    for turn in history:
+        if not turn or len(turn) < 2:
+            continue
+        user_msg, assistant_msg = turn[0], turn[1]
+        lines.append(f"User: {user_msg}")
+        lines.append(f"Assistant: {assistant_msg}")
+    return "\n".join(lines)
+
+
+def _docs_to_loggable(docs, max_chars=220):
+    """Return lightweight doc info for logs (no full dump)."""
+    out = []
+    for d in (docs or []):
+        src = (d.metadata or {}).get("source", "unknown")
+        txt = (d.page_content or "").strip().replace("\n", " ")
+        out.append({
+            "source": src,
+            "preview": (txt[:max_chars] + ("..." if len(txt) > max_chars else "")),
+            "metadata": (d.metadata or {}),
+        })
+    return out
+
+
+def generate_answer(message, history):
+    """
+    Core logic:
+    - rewrite question with history (best-effort)
+    - run RAG (required; if this fails, return a fallback reply)
+    - evaluate (best-effort; if this fails, skip retry)
+    - optionally retry once based on evaluator signal
+    Returns ONLY the final answer string (no sources/context/evaluator in UI).
+    """
+    log_event("request.start", user_message=message)
+
+    # ---------- 1. Rewrite with history (best-effort) ----------
+    try:
+        standalone_question = rewrite_question_with_history(history, message)
+    except Exception as e:
+        log_event("rewrite.error", error=str(e))
+        standalone_question = message  # fallback: use original message
+
+    history_text = _history_to_text(history)
+
+    log_event(
+        "rewrite.done",
+        standalone_question=standalone_question,
+        history_chars=len(history_text),
+    )
+
+    # ---------- 2. Run RAG (if this fails, we bail with generic error text) ----------
+    try:
+        with state_lock:
+            local_rag_chain = rag_chain
+            local_system_prompt = system_prompt
+
+        rag_res = local_rag_chain.invoke({
+            "input": standalone_question,
+            "chat_history": history_text,
+        })
+    except Exception as e:
+        log_event("rag.error", error=str(e))
+        fallback = (
+            "I'm having trouble accessing my knowledge base right now. "
+            "Please try again in a moment."
+        )
+        log_event(
+            "request.end",
+            final_answer_preview=fallback[:400] + ("..." if len(fallback) > 400 else "")
+        )
+        return fallback
+
+    answer_1 = rag_res.get("answer", "") or ""
+    ctx_docs_1 = rag_res.get("context", []) or []
+
+    log_event(
+        "rag.done",
+        answer_preview=answer_1[:400] + ("..." if len(answer_1) > 400 else ""),
+        retrieved_count=len(ctx_docs_1),
+        retrieved_docs=_docs_to_loggable(ctx_docs_1),
+    )
+
+    # ---------- 3. Evaluate (best-effort; never crash on judge failure) ----------
+    eval_res_1 = None
+    try:
+        eval_res_1 = evaluate_answer(
+            system_prompt=local_system_prompt,
+            question=message,
+            context_docs=ctx_docs_1,
+            answer=answer_1,
+        )
+
+        log_event(
+            "eval.done",
+            overall_score=float(eval_res_1.overall_score),
+            grounded=float(eval_res_1.grounded_in_context_score),
+            hallucination=bool(eval_res_1.hallucination_detected),
+            feedback=str(eval_res_1.feedback),
+        )
+    except Exception as e:
+        log_event("eval.error", error=str(e))
+        # We just skip retry logic; answer_1 is still valid.
+
+    final_answer = answer_1
+
+    # ---------- 4. Single retry (only if evaluator succeeded & says to retry) ----------
+    try:
+        if (
+            eval_res_1 is not None and
+           ( eval_res_1.overall_score < 0.70 or getattr(eval_res_1, "should_retry", True))
+        ):
+            revision_prompt = (
+                f"{standalone_question}\n\n"
+                f"You previously answered this:\n{answer_1}\n\n"
+                "An evaluator found issues. Revise your answer to address the feedback below.\n"
+                "Rules:\n"
+                "- Use ONLY the provided context.\n"
+                "- If the context does not support the claim, say \"I don't know\".\n"
+                "- Be specific and grounded.\n\n"
+                f"Evaluator feedback:\n{eval_res_1.feedback}\n"
+            )
+
+            log_event(
+                "retry.triggered",
+                reason="eval_score_below_threshold",
+                threshold=0.90,
+            )
+
+            # RAG retry — if this fails, we keep the original answer_1
+            try:
+                rag_res_2 = rag_chain.invoke({
+                    "input": revision_prompt,
+                    "chat_history": history_text,
+                })
+                answer_2 = rag_res_2.get("answer", "") or ""
+                ctx_docs_2 = rag_res_2.get("context", []) or []
+
+                log_event(
+                    "rag.retry_done",
+                    answer_preview=answer_2[:400] + ("..." if len(answer_2) > 400 else ""),
+                    retrieved_count=len(ctx_docs_2),
+                    retrieved_docs=_docs_to_loggable(ctx_docs_2),
+                )
+
+                # Optional: re-evaluate the revised answer (ignore errors)
+                try:
+                    eval_res_2 = evaluate_answer(
+                        system_prompt=system_prompt,
+                        question=message,
+                        context_docs=ctx_docs_2,
+                        answer=answer_2,
+                    )
+                    log_event(
+                        "eval.retry_done",
+                        overall_score=float(eval_res_2.overall_score),
+                        grounded=float(eval_res_2.grounded_in_context_score),
+                        hallucination=bool(eval_res_2.hallucination_detected),
+                        feedback=str(eval_res_2.feedback),
+                    )
+                except Exception as e_eval2:
+                    log_event("eval.retry_error", error=str(e_eval2))
+
+                # If we got here, second answer is safe to use
+                if eval_res_1.overall_score>eval_res_2.overall_score:
+                    final_answer = answer_1
+                else:
+                    final_answer = answer_2
+
+            except Exception as e_rag2:
+                # Retry RAG failed; log and fall back to first answer
+                log_event("rag.retry_error", error=str(e_rag2))
+                final_answer = answer_1
+
+    except Exception as e_retry_block:
+        # Any unexpected error in retry logic should not crash the whole request
+        log_event("retry.block_error", error=str(e_retry_block))
+        final_answer = answer_1
+
+    # ---------- 5. Final logging & return ----------
+    log_event(
+        "request.end",
+        final_answer_preview=final_answer[:400] + ("..." if len(final_answer) > 400 else "")
+    )
+
+    return final_answer
+
+
+def respond(message, history):
+    """
+    Gradio wrapper that is resilient to unexpected exceptions.
+    If anything explodes inside generate_answer, we log it and return
+    a safe fallback message.
+    """
+    if not message:
+        return "", history
+
+    try:
+        answer = generate_answer(message, history)
+    except Exception as e:
+        log_event("respond.fatal_error", error=str(e))
+        answer = (
+            "Something went wrong on my side while trying to answer. "
+            "Please try again in a moment."
+        )
+
+    history = history + [[message, answer]]
+    return "", history
+
+
+# ---------- Gradio UI ----------
+with gr.Blocks(title="Ask Ritam (Career QA Bot)") as demo:
+    gr.Markdown(
+        "# Ask Ritam\n"
+        "A RAG-powered career assistant over my resume, website, and projects.\n"
+        "Ask anything about my experience, projects, research, or education."
+    )
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            chatbot = gr.Chatbot(label="Conversation", height=500)
+
+            with gr.Row():
+                msg = gr.Textbox(
+                    placeholder="Ask anything about my career, projects, or research...",
+                    lines=2,
+                    scale=4,
+                    show_label=False,
+                )
+                send_btn = gr.Button("Send", variant="primary", scale=1)
+
+            clear_btn = gr.Button("Clear chat")
+
+            send_btn.click(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])
+            msg.submit(respond, inputs=[msg, chatbot], outputs=[msg, chatbot])
+
+            clear_btn.click(lambda: ([], ""), outputs=[chatbot, msg])
+
+demo.launch()

rag_core/FusedRetreiver.py

@@ -0,0 +1,107 @@
+from typing import Any, List, Set, Tuple
+from langchain.schema import BaseRetriever, Document
+
+class FusedRetriever(BaseRetriever):
+    """
+    Pydantic-compatible fused retriever that wraps a header-aware PrefixRetriever
+    and a vector retriever. Declares fields so Pydantic validation succeeds.
+    """
+    prefix_retriever: Any
+    vector_retriever: Any
+    k: int = 4
+    prefix_first: bool = True
+
+    class Config:
+        # allows storing arbitrary Python objects in model fields
+        arbitrary_types_allowed = True
+
+    def get_relevant_documents(self, query: str) -> List[Document]:
+        # 1) prefix candidates
+        prefix_docs = []
+        if hasattr(self.prefix_retriever, "get_relevant_documents"):
+            prefix_docs = self.prefix_retriever.get_relevant_documents(query)
+        elif hasattr(self.prefix_retriever, "_get_relevant_documents"):
+            prefix_docs = self.prefix_retriever._get_relevant_documents(query)
+
+        # 2) vector candidates
+        vector_docs = []
+        try:
+            # many LangChain retrievers implement get_relevant_documents
+            if hasattr(self.vector_retriever, "get_relevant_documents"):
+                vector_docs = self.vector_retriever.get_relevant_documents(query)
+            elif hasattr(self.vector_retriever, "retrieve"):
+                vector_docs = self.vector_retriever.retrieve(query)
+            elif hasattr(self.vector_retriever, "get_relevant_documents_async"):
+                vector_docs = self.vector_retriever.get_relevant_documents_async(query)
+        except Exception:
+            vector_docs = []
+
+        # fuse with dedupe
+        seen: Set[Tuple[str, str]] = set()
+        out: List[Document] = []
+
+        def add_docs(docs: List[Document]) -> bool:
+            for d in docs:
+                key = (d.metadata.get("source"), d.page_content[:200])
+                if key in seen:
+                    continue
+                seen.add(key)
+                out.append(d)
+                if len(out) >= self.k:
+                    return True
+            return False
+
+        if self.prefix_first:
+            finished = add_docs(prefix_docs)
+            if not finished:
+                add_docs(vector_docs)
+        else:
+            finished = add_docs(vector_docs)
+            if not finished:
+                add_docs(prefix_docs)
+
+        return out[: self.k]
+
+    async def aget_relevant_documents(self, query: str) -> List[Document]:
+        # Try to call async variants when available, else fall back to sync
+        prefix_docs = []
+        if hasattr(self.prefix_retriever, "aget_relevant_documents"):
+            prefix_docs = await self.prefix_retriever.aget_relevant_documents(query)
+        elif hasattr(self.prefix_retriever, "get_relevant_documents"):
+            prefix_docs = self.prefix_retriever.get_relevant_documents(query)
+        elif hasattr(self.prefix_retriever, "_get_relevant_documents"):
+            prefix_docs = self.prefix_retriever._get_relevant_documents(query)
+
+        vector_docs = []
+        if hasattr(self.vector_retriever, "aget_relevant_documents"):
+            vector_docs = await self.vector_retriever.aget_relevant_documents(query)
+        elif hasattr(self.vector_retriever, "get_relevant_documents"):
+            vector_docs = self.vector_retriever.get_relevant_documents(query)
+        elif hasattr(self.vector_retriever, "retrieve"):
+            vector_docs = self.vector_retriever.retrieve(query)
+
+        # Reuse sync fuse logic by delegating to get_relevant_documents after patching
+        # Build a temporary 'self' like structure — easiest is to merge candidate lists here
+        seen = set()
+        out = []
+        def add_docs_sync(docs):
+            for d in docs:
+                key = (d.metadata.get("source"), d.page_content[:200])
+                if key in seen:
+                    continue
+                seen.add(key)
+                out.append(d)
+                if len(out) >= self.k:
+                    return True
+            return False
+
+        if self.prefix_first:
+            finished = add_docs_sync(prefix_docs)
+            if not finished:
+                add_docs_sync(vector_docs)
+        else:
+            finished = add_docs_sync(vector_docs)
+            if not finished:
+                add_docs_sync(prefix_docs)
+
+        return out[: self.k]

rag_core/PrefixRetreiver.py

@@ -0,0 +1,59 @@
+from typing import List, Optional
+from langchain_core.documents import Document
+from langchain_core.retrievers import BaseRetriever
+from pydantic import Field
+from langchain.schema import Document
+import re
+
+def _norm(text: str) -> str:
+    # lower, remove punctuation-ish chars, compact whitespace
+    text = text or ""
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+class PrefixRetriever:
+    def __init__(self, docs: List[Document], k: int = 3, max_lines: int = 8):
+        self.docs = docs
+        self.k = k
+        self.max_lines = max_lines
+
+    def _head(self, content: str) -> str:
+        # return up to max_lines lines from the beginning to be used for prefix matching
+        lines = [ln for ln in content.splitlines() if ln.strip()]
+        return "\n".join(lines[: self.max_lines])
+
+    def _get_relevant_documents(self, query: str) -> List[Document]:
+        q = _norm(query)
+        tokens = [t for t in q.split() if t]
+        out: List[Document] = []
+
+        for d in self.docs:
+            # build a matching head: prefer explicit header metadata
+            header = d.metadata.get("section_header") or d.metadata.get("section_label") or ""
+            head_text = header + "\n" + self._head(d.page_content)
+            head = _norm(head_text)
+
+            matched = False
+            # 1) exact-substring match (most important)
+            if q and q in head:
+                matched = True
+            else:
+                # 2) token-based match: ensure every token exists in head
+                if tokens and all(tok in head for tok in tokens):
+                    matched = True
+
+            if matched:
+                out.append(d)
+                if len(out) >= self.k:
+                    break
+
+        return out
+
+
+# --- Helper to recover Documents from a FAISS vectorstore
+def faiss_all_docs(faiss_store):
+    ids = list(faiss_store.index_to_docstore_id.values())
+    return [faiss_store.docstore.search(_id) for _id in ids]
+

rag_core/config.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+DATA_DIR = BASE_DIR / "data"
+VECTORSTORE_DIR = DATA_DIR / "vectorstore"
+
+VECTORSTORE_DIR.mkdir(parents=True, exist_ok=True)
+
+VECTORSTORE_PATH = str(VECTORSTORE_DIR / "career_faiss_index")
+
+GROQ_CHAT_MODEL = "meta-llama/llama-4-scout-17b-16e-instruct"
+GROQ_EVAL_MODEL = "llama-3.1-8b-instant"
+EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"

rag_core/crawler.py

@@ -0,0 +1,54 @@
+# rag_core/crawler.py
+from urllib.parse import urljoin, urlparse
+from collections import deque
+import requests
+from bs4 import BeautifulSoup
+
+def crawl_subpages(root_url: str, max_pages: int = 30, max_depth: int = 1):
+    """
+    Crawl subpages under the same path as root_url.
+    E.g., root_url = https://site.com/projects/
+    Will collect https://site.com/projects/* links.
+    """
+    parsed_root = urlparse(root_url)
+    base_domain = parsed_root.netloc
+    base_path = parsed_root.path.rstrip("/")  # "/projects"
+
+    visited = set()
+    to_visit = deque([(root_url, 0)])
+    collected = set([root_url])
+
+    while to_visit and len(collected) < max_pages:
+        url, depth = to_visit.popleft()
+        if url in visited or depth > max_depth:
+            continue
+        visited.add(url)
+
+        try:
+            resp = requests.get(url, timeout=10)
+            if not resp.ok:
+                continue
+            soup = BeautifulSoup(resp.text, "html.parser")
+        except Exception:
+            continue
+
+        for a in soup.find_all("a", href=True):
+            href = a["href"].strip()
+            full_url = urljoin(url, href)
+            parsed = urlparse(full_url)
+
+            # Stay in same domain
+            if parsed.netloc != base_domain:
+                continue
+
+            # Stay under the same base path
+            if not parsed.path.startswith(base_path):
+                continue
+
+            if full_url not in collected:
+                collected.add(full_url)
+                to_visit.append((full_url, depth + 1))
+
+        visited.add(url)
+
+    return list(collected)

rag_core/embeddings_model.py

@@ -0,0 +1,18 @@
+# rag_core/embeddings_model.py
+from dotenv import load_dotenv
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from .config import EMBEDDING_MODEL
+
+load_dotenv()  # loads HUGGINGFACEHUB_API_TOKEN if present
+
+def get_embeddings():
+    """
+    Returns a HuggingFaceEmbeddings instance using all-MiniLM.
+    This runs locally via sentence-transformers and will download
+    the model from Hugging Face (using your HF token if needed).
+    """
+    return HuggingFaceEmbeddings(
+        model_name=EMBEDDING_MODEL,
+        model_kwargs={"device": "cpu"},      # change to "cuda" if you have a GPU
+        encode_kwargs={"normalize_embeddings": True},  # optional but often helpful
+    )

rag_core/evaluator.py

@@ -0,0 +1,85 @@
+# rag_core/evaluator.py
+from langchain_core.prompts import ChatPromptTemplate
+
+from .models_groq import get_judge_llm
+from .evaluator_schema import EvalResult
+
+# Base LLM
+_base_judge_llm = get_judge_llm()
+
+# Wrap LLM with structured output (EvalResult)
+_judge_llm_structured = _base_judge_llm.with_structured_output(EvalResult)
+
+# Prompt template for the judge
+JUDGE_PROMPT = ChatPromptTemplate.from_messages(
+    [
+        ("system", """
+You are an impartial evaluator for a RAG-based assistant about Ritam's career.
+
+You get:
+- System prompt,
+- User question,
+- Retrieved context,
+- Assistant answer.
+
+Check:
+1. Does the answer follow the system prompt? (only Ritam's career / projects / research.)
+2. Does the answer actually respond to the user's question?
+3. Is the answer having too much extra information
+3. How well is the answer grounded in the context? Penalize hallucinations.
+
+Return a strict JSON object matching the EvalResult schema.
+"""),
+        ("human", """
+System prompt:
+----------------
+{system_prompt}
+
+User question:
+----------------
+{question}
+
+Retrieved context:
+----------------
+{context}
+
+Assistant answer:
+----------------
+{answer}
+"""),
+    ]
+)
+
+def evaluate_answer(system_prompt: str, question: str, context_docs, answer: str) -> EvalResult:
+    """
+    Run the LLM judge with structured output over the given answer.
+    If the judge fails, return a safe EvalResult that disables retry.
+    """
+    ctx_text = "\n\n".join(
+        f"[DOC {i}] (source={d.metadata.get('source', 'unknown')})\n{d.page_content}"
+        for i, d in enumerate(context_docs)
+    )
+
+    chain = JUDGE_PROMPT | _judge_llm_structured
+
+    try:
+        result: EvalResult = chain.invoke({
+            "system_prompt": system_prompt,
+            "question": question,
+            "context": ctx_text,
+            "answer": answer,
+        })
+        return result
+
+    except Exception as e:
+        # Judge died / timed out / bad response
+        # -> don't retry, but don't crash the main answer either.
+        return EvalResult(
+            follows_system_prompt=True,           # neutral / optimistic defaults
+            answers_user_question=True,
+            grounded_in_context_score=1.0,
+            hallucination_detected=False,
+            overall_score=1.0,                   # high score so threshold logic won't trigger
+            should_retry=False,                  # explicitly NO RETRY
+            feedback=f"[Evaluator failure] Judge could not evaluate this answer: {e}",
+        )

rag_core/evaluator_schema.py

@@ -0,0 +1,11 @@
+# rag_core/evaluator_schema.py
+from langchain_core.pydantic_v1 import BaseModel, Field
+
+class EvalResult(BaseModel):
+    follows_system_prompt: bool = Field(...)
+    answers_user_question: bool = Field(...)
+    grounded_in_context_score: float = Field(ge=0.0, le=1.0)
+    hallucination_detected: bool = Field(...)
+    overall_score: float = Field(ge=0.0, le=1.0)
+    should_retry: bool = Field(...)
+    feedback: str = Field(...)

rag_core/index_builder.py

@@ -0,0 +1,300 @@
+# rag_core/index_builder.py
+
+import re
+from typing import List
+
+import requests
+from langchain_community.document_loaders import OnlinePDFLoader
+from langchain_text_splitters import HTMLSectionSplitter, RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+
+from .embeddings_model import get_embeddings
+from .config import VECTORSTORE_PATH
+from .sources import CRAWL_ROOTS, FIXED_URLS
+from .crawler import crawl_subpages
+
+
+def _is_gdrive_file(url: str) -> bool:
+    """Return True if this looks like a Google Drive file view URL."""
+    return "drive.google.com" in url and "/file/d/" in url
+
+
+def _gdrive_view_to_download(url: str) -> str:
+    """
+    Convert a Google Drive view URL to a direct download URL.
+
+    Example:
+      https://drive.google.com/file/d/<ID>/view
+      -> https://drive.google.com/uc?export=download&id=<ID>
+    """
+    m = re.search(r"/file/d/([^/]+)/", url)
+    if not m:
+        return url
+    file_id = m.group(1)
+    return f"https://drive.google.com/uc?export=download&id={file_id}"
+
+
+def _infer_section_label_from_url(url: str) -> str:
+    """
+    Heuristic: guess a section label from the URL path.
+    e.g.
+      https://your-site.com/about                 -> 'about'
+      https://your-site.com/experience/juniper   -> 'experience/juniper'
+    """
+    try:
+        path = url.split("://", 1)[-1].split("/", 1)[-1]
+    except Exception:
+        return url
+    path = path.strip("/")
+    if not path:
+        return "root"
+    return path
+
+
+def _normalize_label(text: str, max_len: int = 80) -> str:
+    """
+    Create a short normalized label from a header text.
+    - lowercases, removes newlines, collapses whitespace
+    - trims to max_len and replaces spaces with '-' for compact labels
+    """
+    if not text:
+        return ""
+    lab = " ".join(text.split())  # collapse whitespace/newlines
+    lab = lab.strip().lower()
+    # shorten if too long
+    if len(lab) > max_len:
+        lab = lab[: max_len - 3].rstrip() + "..."
+    # use a compact label form (slug-like) but keep readability
+    slug = lab.replace(" ", "-")
+    # remove characters that would be awkward in metadata keys
+    slug = re.sub(r"[^a-z0-9_\-\.]+", "", slug)
+    return slug
+
+
+def load_web_docs(urls: List[str]):
+    """
+    Load documents from a list of URLs.
+
+    - HTML URLs:
+        * Fetch raw HTML with `requests`
+        * Split into sections with HTMLSectionSplitter (h1/h2)
+        * Infer section labels from the actual section header lines (preferred)
+    - PDF URLs (including Google Drive file links):
+        * Load with OnlinePDFLoader (one doc per page)
+    """
+    html_urls: List[str] = []
+    pdf_urls: List[str] = []
+
+    for url in urls:
+        u = url.strip()
+        if not u:
+            continue
+
+        # Google Drive file links (view) -> direct download PDF
+        if _is_gdrive_file(u):
+            pdf_urls.append(_gdrive_view_to_download(u))
+        # Direct PDF URLs
+        elif u.lower().endswith(".pdf"):
+            pdf_urls.append(u)
+        # Everything else is treated as HTML
+        else:
+            html_urls.append(u)
+
+    docs = []
+
+    # --- HTML via HTMLSectionSplitter on raw HTML string ---
+    if html_urls:
+        print(f"[index_builder] Loading HTML for {len(html_urls)} URLs with HTMLSectionSplitter")
+
+        headers_to_split_on = [
+            ("h1", "Header 1"),
+            ("h2", "Header 2"),
+        ]
+        html_splitter = HTMLSectionSplitter(headers_to_split_on=headers_to_split_on)
+
+        for url in html_urls:
+            try:
+                print(f"[index_builder]   Fetching HTML from {url}")
+                resp = requests.get(url, timeout=15)
+                resp.raise_for_status()
+                html_string = resp.text
+
+                # HTMLSectionSplitter returns a list[Document] where each Document starts with the header line
+                html_header_splits = html_splitter.split_text(html_string)
+
+                # we will prefer to use the header line as the canonical label for each section
+                # but ensure we deduplicate very similar headers within the same page
+                seen_labels_in_page = set()
+
+                print(f"[index_builder]   {url}: {len(html_header_splits)} HTML sections")
+
+                for d in html_header_splits:
+                    # source URL
+                    d.metadata["source"] = url
+                    # extract the first meaningful non-empty line as the header
+                    first_lines = [ln.strip() for ln in d.page_content.splitlines() if ln.strip()]
+                    header_line = first_lines[0] if first_lines else ""
+
+                    # normalize header_line for metadata and label
+                    # keep section_header as human readable short header (truncated if necessary)
+                    human_header = header_line
+                    if len(human_header) > 300:
+                        human_header = human_header[:300] + "..."
+
+                    if human_header != "More Project":
+                        d.metadata["section_header"] = human_header
+                    else:
+                        # Extract endpoint from URL as fallback header
+                        try:
+                            # e.g. https://site.com/projects/my-cool-project -> "my-cool-project"
+                            endpoint = url.rstrip("/").split("/")[-1]
+                            # Make it human-readable
+                            endpoint = endpoint.replace("-", " ").replace("_", " ").title()
+                            d.metadata["section_header"] = endpoint
+                            d.metadata["section_label"] = endpoint
+                        except Exception:
+                            # absolute fallback
+                            d.metadata["section_header"] = human_header
+                            d.metadata["section_label"] = human_header
+
+                    # produce a short machine-friendly label from header; fallback to URL-based label
+                    # label_from_header = _normalize_label(header_line)
+                    # if not label_from_header:
+                    #     label_from_header = _infer_section_label_from_url(url)
+
+                    # # dedupe labels within this page (if splitter produced repeated headers)
+                    # dedup_label = label_from_header
+                    # suffix = 1
+                    # while dedup_label in seen_labels_in_page:
+                    #     dedup_label = f"{label_from_header}-{suffix}"
+                    #     suffix += 1
+                    # seen_labels_in_page.add(dedup_label)
+
+                    # d.metadata["section_label"] = dedup_label
+                    d.metadata["section_type"] = "remote_html"
+                # append docs
+                docs.extend(html_header_splits)
+            except Exception as e:
+                print(f"[index_builder] Error processing HTML from {url}: {e}")
+
+    # --- PDF files (including Drive) via OnlinePDFLoader ---
+    for pdf_url in pdf_urls:
+        print(f"[index_builder] Loading PDF from {pdf_url}")
+        try:
+            pdf_loader = OnlinePDFLoader(pdf_url)
+            pdf_docs = pdf_loader.load()
+            section_label = _infer_section_label_from_url(pdf_url)
+            for d in pdf_docs:
+                d.metadata["source"] = pdf_url
+                d.metadata["section_label"] = section_label
+                d.metadata["section_type"] = "remote_pdf"
+            docs.extend(pdf_docs)
+            print(f"[index_builder]   {pdf_url}: {len(pdf_docs)} PDF pages")
+        except Exception as e:
+            print(f"[index_builder] Failed to load PDF from {pdf_url}: {e}")
+
+    return docs
+
+
+def split_docs(docs, chunk_size: int = 1000, chunk_overlap: int = 200):
+    """
+    Split loaded documents into chunks for embedding.
+
+    - HTML docs (from HTMLSectionSplitter) are already section-level chunks → keep as-is.
+    - Non-HTML docs (PDF pages, etc.) are split with RecursiveCharacterTextSplitter.
+    """
+    html_docs = [d for d in docs if d.metadata.get("section_type") == "remote_html"]
+    other_docs = [d for d in docs if d.metadata.get("section_type") != "remote_html"]
+
+    chunks: List = []
+
+    # Keep HTML sections as they are
+    chunks.extend(html_docs)
+
+    # Split other docs (PDFs, etc.) into text chunks
+    if other_docs:
+        splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            add_start_index=True,
+        )
+        other_chunks = splitter.split_documents(other_docs)
+        for c in other_chunks:
+            c.metadata.setdefault("section_label", c.metadata.get("source", "unknown"))
+            c.metadata.setdefault(
+                "section_type",
+                c.metadata.get("section_type", "remote_pdf"),
+            )
+        chunks.extend(other_chunks)
+
+    print(
+        f"[index_builder] split_docs: {len(html_docs)} HTML section chunks, "
+        f"{len(chunks) - len(html_docs)} non-HTML chunks"
+    )
+    return chunks
+
+
+def build_and_save_index():
+    """
+    Crawl URLs, load docs, split into chunks, build FAISS index, and save it.
+
+    Returns:
+        (int, list): number of chunks indexed, and the chunks themselves.
+    """
+    # 1. Crawl project roots (and any other CRAWL_ROOTS) to get sub-URLs
+    crawl_urls: List[str] = []
+    for root in CRAWL_ROOTS:
+        try:
+            urls = crawl_subpages(root)
+            print(f"[index_builder] Crawled {len(urls)} URLs under {root}")
+            crawl_urls.extend(urls)
+        except Exception as e:
+            print(f"[index_builder] Failed to crawl {root}: {e}")
+
+    # 2. Combine fixed URLs (resume, about, scholar, GitHub, etc.) + crawled URLs
+    all_urls = list(set(FIXED_URLS + crawl_urls))
+
+    print(f"[index_builder] Total URLs to load: {len(all_urls)}")
+    for u in all_urls:
+        print(f"  - {u}")
+
+    docs = load_web_docs(all_urls)
+    print(f"[index_builder] Loaded {len(docs)} raw documents")
+
+    if not docs:
+        print("[index_builder] WARNING: No documents loaded; aborting index build.")
+        return 0, []
+
+    # 3. Split into chunks (HTML via HTMLSectionSplitter, PDFs via recursive splitter)
+    chunks = split_docs(docs)
+    print(f"[index_builder] Split into {len(chunks)} chunks")
+
+    if not chunks:
+        print("[index_builder] WARNING: No chunks produced; aborting FAISS build.")
+        return 0, []
+
+    # 4. Build vector store with HF embeddings (e.g., all-MiniLM)
+    embeddings = get_embeddings()
+    vs = FAISS.from_documents(chunks, embeddings)
+
+    # 5. Save to disk
+    vs.save_local(VECTORSTORE_PATH)
+    print(
+        f"[index_builder] Saved FAISS index to {VECTORSTORE_PATH} "
+        f"(chunks={len(chunks)})"
+    )
+
+    return len(chunks), chunks
+
+
+def load_vectorstore():
+    """
+    Load the FAISS vector store from disk using the same embedding model.
+    """
+    embeddings = get_embeddings()
+    vs = FAISS.load_local(
+        VECTORSTORE_PATH,
+        embeddings,
+        allow_dangerous_deserialization=True,
+    )
+    return vs

rag_core/models_groq.py

@@ -0,0 +1,15 @@
+# rag_core/models_groq.py
+from dotenv import load_dotenv
+from langchain_core.callbacks import Callbacks
+from langchain_core.caches import BaseCache
+from langchain_groq import ChatGroq
+from .config import GROQ_CHAT_MODEL, GROQ_EVAL_MODEL
+
+load_dotenv()
+ChatGroq.model_rebuild()
+
+def get_answer_llm():
+    return ChatGroq(model=GROQ_CHAT_MODEL, temperature=0.1)
+
+def get_judge_llm():
+    return ChatGroq(model=GROQ_EVAL_MODEL, temperature=0.0)

rag_core/rag_chain.py

@@ -0,0 +1,200 @@
+# rag_core/rag_chain.py (top of file)
+
+from typing import List, Tuple, Dict, Any
+from langchain.schema import Document, BaseRetriever
+from transformers import pipeline
+from rag_core.index_builder import load_vectorstore, build_and_save_index
+
+
+# ---- Label routing helpers ----
+
+# Global zero-shot classifier (loaded once)
+label_classifier = pipeline(
+    "zero-shot-classification",
+    model="facebook/bart-large-mnli"
+)
+
+def build_label_vocab(docs: List[Document]) -> List[str]:
+    labels = []
+    seen = set()
+    for d in docs:
+        header = (d.metadata.get("section_header") or "").strip()
+        s_label = (d.metadata.get("section_label") or "").strip()
+        candidates = [header, s_label]
+        for c in candidates:
+            if not c:
+                continue
+            normalized = " ".join(c.split())
+            if len(normalized) > 120:
+                normalized = normalized[:120] + "..."
+            if normalized not in seen:
+                seen.add(normalized)
+                labels.append(normalized)
+    return labels
+
+def map_query_to_labels_zero_shot(
+    query: str,
+    candidate_labels: List[str],
+    top_k: int = 5,
+    score_threshold: float = 0.40,
+) -> List[Tuple[str, float]]:
+    if not candidate_labels:
+        return []
+    out = label_classifier(query, candidate_labels, multi_label=True)
+    labels_out = out["labels"]
+    scores_out = out["scores"]
+    selected: List[Tuple[str, float]] = []
+    for lbl, score in zip(labels_out[:top_k], scores_out[:top_k]):
+        if score >= score_threshold:
+            selected.append((lbl, float(score)))
+    if not selected and labels_out:
+        selected = [(labels_out[0], float(scores_out[0]))]
+    return selected
+
+def fetch_docs_by_labels_with_scores(
+    selected_labels: List[Tuple[str, float]],
+    docs: List[Document],
+) -> List[Tuple[Document, float, List[str]]]:
+    """
+    For each doc, determine which of selected_labels it matches
+    (substring match on section_header / section_label).
+    Return (Document, score, [matched_labels]) where score is max label score.
+    """
+    if not selected_labels:
+        return []
+    label_score: Dict[str, float] = {lbl.lower(): sc for lbl, sc in selected_labels}
+    out: List[Tuple[Document, float, List[str]]] = []
+    for d in docs:
+        header = (d.metadata.get("section_header") or "").lower()
+        s_label = (d.metadata.get("section_label") or "").lower()
+        combined = header + " " + s_label
+
+        matched_labels: List[str] = []
+        matched_scores: List[float] = []
+        for lbl_lower, sc in label_score.items():
+            if lbl_lower and lbl_lower in combined:
+                matched_labels.append(lbl_lower)
+                matched_scores.append(sc)
+
+        if matched_labels:
+            doc_score = max(matched_scores)
+            out.append((d, doc_score, matched_labels))
+    return out
+
+class LabelRoutingRetriever(BaseRetriever):
+    """
+    Retriever that:
+      1) Uses BART-MNLI to map query -> section labels.
+      2) Fetches all docs whose header/label match those labels.
+      3) Ranks docs by label confidence.
+      4) Falls back to vector retriever if no labels match.
+    """
+
+    docs: List[Document]
+    vector_retriever: Any = None
+    top_k_labels: int = 5
+    label_score_threshold: float = 0.35
+    k_docs: int = 6
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def __init__(
+        self,
+        docs: List[Document],
+        vector_retriever: Any = None,
+        top_k_labels: int = 5,
+        label_score_threshold: float = 0.35,
+        k_docs: int = 6,
+        **kwargs,
+    ):
+        super().__init__(
+            docs=docs,
+            vector_retriever=vector_retriever,
+            top_k_labels=top_k_labels,
+            label_score_threshold=label_score_threshold,
+            k_docs=k_docs,
+            **kwargs,
+        )
+
+    def get_relevant_documents(self, query: str) -> List[Document]:
+        # 1) build candidate labels from docs
+        candidate_labels = build_label_vocab(self.docs)
+
+        # 2) map query -> (label, score)
+        mapped = map_query_to_labels_zero_shot(
+            query,
+            candidate_labels,
+            top_k=self.top_k_labels,
+            score_threshold=self.label_score_threshold,
+        )
+
+        # 3) fetch docs with scores
+        docs_with_scores = fetch_docs_by_labels_with_scores(mapped, self.docs)
+
+        if not docs_with_scores and self.vector_retriever is not None:
+            # fallback to vector retriever (semantic retrieval)
+            vec_docs = self.vector_retriever.get_relevant_documents(query)
+            return vec_docs[: self.k_docs]
+
+        # 4) sort by score desc + dedupe
+        seen_keys = set()
+        ranked_docs: List[Document] = []
+        for d, sc, matched in sorted(docs_with_scores, key=lambda x: x[1], reverse=True):
+            if d.metadata.get("Header 2")=='More Project':
+                continue
+            key = (d.metadata.get("source"), d.page_content[:200])
+            if key in seen_keys:
+                continue
+            seen_keys.add(key)
+            ranked_docs.append(d)
+            if len(ranked_docs) >= self.k_docs:
+                break
+        return ranked_docs
+
+    async def aget_relevant_documents(self, query: str) -> List[Document]:
+        # simple async wrapper
+        return self.get_relevant_documents(query)
+
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.prompts import ChatPromptTemplate
+from langchain.chains import create_retrieval_chain
+from .models_groq import get_answer_llm, get_judge_llm
+
+SYSTEM_PROMPT = """You are Ritam's personal QA bot.
+                Use the following context from his website and resume to answer.
+
+                Question: {input}
+                Context:
+                {context}
+
+                Answer in first person as Ritam."""
+
+def build_rag_chain(vs, k=5, max_docs=6):
+    
+    all_docs = list(vs.docstore._dict.values())
+    vector_retriever = vs.as_retriever(search_kwargs={"k": 8})
+
+    # old: ensemble/PrefixRetriever
+    # new: label routing retriever
+    retriever = LabelRoutingRetriever(
+        docs=all_docs,
+        vector_retriever=vector_retriever,
+        top_k_labels=k,
+        label_score_threshold=0.4,
+        k_docs=max_docs,
+    )
+
+    prompt = ChatPromptTemplate.from_template(
+            SYSTEM_PROMPT
+            )
+    
+    llm = get_answer_llm()
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", SYSTEM_PROMPT),
+        ("human", "{input}"),
+    ])
+
+    combine_docs_chain = create_stuff_documents_chain(llm, prompt)
+    rag_chain = create_retrieval_chain(retriever, combine_docs_chain)
+    return rag_chain, retriever, SYSTEM_PROMPT

rag_core/rag_chain_helper.py

@@ -0,0 +1,137 @@
+# rag_core/rag_chain.py
+from langchain.chains import create_retrieval_chain
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain_core.prompts import ChatPromptTemplate
+from rag_core.FusedRetreiver import FusedRetriever
+from rag_core.PrefixRetreiver import PrefixRetriever
+
+
+from .models_groq import get_answer_llm, get_judge_llm
+
+def faiss_all_docs(faiss_store):
+    # Map FAISS index positions -> docstore ids
+    ids = list(faiss_store.index_to_docstore_id.values())
+    # Pull Documents from the docstore in that order
+    return [faiss_store.docstore.search(_id) for _id in ids]
+
+
+SYSTEM_PROMPT = """
+You are a QA assistant for Ritam that answers questions about Ritam's career posing as Ritam's digital twin.
+
+You will receive:
+- `context`: text chunks retrieved from Ritam's resume, website, project pages, scholar profile, etc.
+- `chat_history`: a short summary of the prior conversation turns for coherence.
+
+Context:
+{context}
+
+Chat history (for reference):
+{chat_history}
+
+Rules:
+- Use ONLY the provided context for factual claims.
+- If the answer is not clearly supported by the context, say "I don't know"
+  and suggest rephrasing in terms of Ritam's work, projects, or research.
+- If the user asks generic questions unrelated to Ritam, politely refuse saying that please ask only career related questions to me.
+
+The chat_history is only for understanding the follow-up question; do NOT invent facts
+that aren't in the context even if they appear earlier in chat_history.
+
+"""
+
+def build_rag_chain(vectorstore, k=2, max_lines=3, weights=(0.35, 0),top_k=None):
+    """
+    Returns (rag_chain, retriever, SYSTEM_PROMPT)
+
+    - vectorstore: your FAISS/Chroma/… already loaded
+    - k: per-retriever fetch size (each retriever will pull up to k docs)
+    - max_lines: how many leading lines to consider for prefix matching
+    - weights: contribution of (Prefix, Vector) retrievers in the fusion
+    - top_k: final number of docs returned by the ensemble (defaults to k)
+    """
+    # ---- Retrievers
+    all_docs = faiss_all_docs(vectorstore)
+    prefix_retriever = PrefixRetriever(docs=all_docs, k=2, max_lines=3)
+
+    # Pull a few extra from vectorstore to give the fusion more to work with
+    vector_retriever = vectorstore.as_retriever(search_kwargs={"k": max(k, 1)})
+
+    fused_retriever = FusedRetriever(
+                        prefix_retriever=prefix_retriever,
+                        vector_retriever=vector_retriever,
+                        k=6,
+                        prefix_first=True,
+                    )
+
+    # ---- LLM + prompt
+    llm = get_answer_llm()
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", SYSTEM_PROMPT),
+        ("human", "{input}"),
+    ])
+
+    # ---- Stuff docs into the prompt, then make the RAG chain
+    document_chain = create_stuff_documents_chain(llm, prompt)
+    rag_chain = create_retrieval_chain(fused_retriever, document_chain)
+
+    return rag_chain, fused_retriever, SYSTEM_PROMPT
+
+
+
+# ---------- Conversational question rewriter ----------
+
+# We’ll use a low-temperature model (judge_llm) to rewrite follow-up questions
+_rewriter_llm = get_judge_llm()
+
+QUESTION_REWRITE_PROMPT = ChatPromptTemplate.from_messages(
+    [
+        ("system", """
+You are a helpful assistant that rewrites follow-up questions into standalone questions.
+
+You are given a chat history between a user and an assistant, plus the user's new question.
+Your job is to rewrite the new question so that it is self-contained and can be understood
+without the previous turns.
+
+The rewritten question MUST stay faithful to the user's intent and be about Ritam's
+career, projects, research, or education.
+If the question is already standalone, return it as-is.
+"""),
+        ("human", """
+Chat history:
+------------
+{chat_history}
+
+New user question:
+------------
+{question}
+
+Rewrite the new question into a single, self-contained question:
+"""),
+    ]
+)
+
+def rewrite_question_with_history(history, question: str) -> str:
+    """
+    history: list of [user_msg, assistant_msg] pairs (from Gradio ChatInterface)
+    question: current user string
+
+    Returns a standalone question string that incorporates context from history.
+    """
+    # If no history, just return the question
+    if not history:
+        return question
+
+    # Convert history to a single text block
+    history_lines = []
+    for turn in history:
+        if not turn or len(turn) < 2:
+            continue
+        user_msg, assistant_msg = turn[0], turn[1]
+        history_lines.append(f"User: {user_msg}")
+        history_lines.append(f"Assistant: {assistant_msg}")
+    history_text = "\n".join(history_lines)
+
+    chain = QUESTION_REWRITE_PROMPT | _rewriter_llm
+    resp = chain.invoke({"chat_history": history_text, "question": question})
+    standalone = resp.content.strip()
+    return standalone or question

rag_core/sources.py

@@ -0,0 +1,25 @@
+# rag_core/sources.py
+
+# Resume URL (HTML or PDF, both okay; PDF will be text-extracted)
+LINKEDIN_URL = "https://www.linkedin.com/in/ritam-upadhyay-51ba81192/"
+
+# Root projects page that links to each project subpage
+PROJECTS_ROOT_URL = "https://fearless-writers-028990.framer.app/project"
+
+# Other URLs directly relevant to your career
+OTHER_CAREER_URLS = [
+    "https://fearless-writers-028990.framer.app/old-home",
+    "https://fearless-writers-028990.framer.app/",
+    "https://scholar.google.com/citations?user=04o0bdcAAAAJ&hl=en",
+    "https://fearless-writers-028990.framer.app/stack",
+]
+
+# These roots will be crawled:
+CRAWL_ROOTS = [
+   PROJECTS_ROOT_URL,
+]
+
+# These are direct, non-crawling URLs:
+FIXED_URLS = [
+    *OTHER_CAREER_URLS,
+]

requirements.txt

@@ -0,0 +1,14 @@
+gradio==5.49.1
+langchain==0.3.12
+python-dotenv==1.0.1
+langchain-core==0.3.28
+langchain-openai==0.2.12
+langchain-community==0.3.12
+langchain-text-splitters==0.3.4
+langchain-groq==0.2.1
+sentence-transformers==3.3.1
+faiss-cpu==1.9.0.post1
+beautifulsoup4==4.12.3
+requests==2.32.3
+pypdf==5.1.0
+lxml==6.0.2