refactoring the code and adding mcp

README.md

@@ -5,7 +5,7 @@ colorFrom: indigo
 colorTo: yellow
 sdk: gradio
 sdk_version: 5.49.1
-app_file: app.py
+app_file: src/app.py
 pinned: false
 python_version: 3.13
 ---

requirements.txt

@@ -1,150 +1,426 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml -o hierRAG/requirements.txt
 aiofiles==24.1.0
+    # via gradio
 aiohappyeyeballs==2.6.1
-aiohttp==3.13.1
+    # via aiohttp
+aiohttp==3.13.2
+    # via langchain-community
 aiosignal==1.4.0
+    # via aiohttp
 annotated-doc==0.0.3
+    # via fastapi
 annotated-types==0.7.0
+    # via pydantic
 anyio==4.11.0
-asttokens==3.0.0
+    # via
+    #   gradio
+    #   httpx
+    #   mcp
+    #   openai
+    #   sse-starlette
+    #   starlette
 attrs==25.4.0
+    # via
+    #   aiohttp
+    #   jsonschema
+    #   referencing
 audioop-lts==0.2.2
+    # via gradio
 brotli==1.1.0
+    # via gradio
 cachetools==6.2.1
+    # via google-auth
 certifi==2025.10.5
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
 cffi==2.0.0
+    # via cryptography
 charset-normalizer==3.4.4
+    # via
+    #   pdfminer-six
+    #   requests
 click==8.3.0
-comm==0.2.3
+    # via
+    #   typer
+    #   typer-slim
+    #   uvicorn
 cryptography==46.0.3
+    # via pdfminer-six
 dataclasses-json==0.6.7
-debugpy==1.8.17
-decorator==5.2.1
+    # via langchain-community
 distro==1.9.0
-executing==2.2.1
-fastapi==0.120.1
+    # via openai
+fastapi==0.120.4
+    # via gradio
 ffmpy==0.6.4
+    # via gradio
 filelock==3.20.0
+    # via huggingface-hub
 filetype==1.2.0
+    # via langchain-google-genai
 frozenlist==1.8.0
-fsspec==2025.9.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2025.10.0
+    # via
+    #   gradio-client
+    #   huggingface-hub
 google-ai-generativelanguage==0.9.0
-google-api-core==2.28.0
-google-auth==2.41.1
+    # via langchain-google-genai
+google-api-core==2.28.1
+    # via google-ai-generativelanguage
+google-auth==2.42.1
+    # via
+    #   google-ai-generativelanguage
+    #   google-api-core
 googleapis-common-protos==1.71.0
+    # via
+    #   google-api-core
+    #   grpcio-status
 gradio==5.49.1
+    # via hier-rag (pyproject.toml)
 gradio-client==1.13.3
+    # via gradio
 greenlet==3.2.4
+    # via sqlalchemy
 groovy==0.1.2
+    # via gradio
 grpcio==1.76.0
+    # via
+    #   google-ai-generativelanguage
+    #   google-api-core
+    #   grpcio-status
+    #   pymilvus
 grpcio-status==1.76.0
+    # via google-api-core
 h11==0.16.0
+    # via
+    #   httpcore
+    #   uvicorn
 hf-xet==1.2.0
+    # via huggingface-hub
 httpcore==1.0.9
+    # via httpx
 httpx==0.28.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   langgraph-sdk
+    #   langsmith
+    #   mcp
+    #   openai
+    #   safehttpx
 httpx-sse==0.4.3
-huggingface-hub==1.0.0
+    # via
+    #   langchain-community
+    #   mcp
+huggingface-hub==1.0.1
+    # via
+    #   gradio
+    #   gradio-client
 idna==3.11
-iniconfig==2.3.0
-ipykernel==7.1.0
-ipython==9.6.0
-ipython-pygments-lexers==1.1.1
-jedi==0.19.2
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
 jinja2==3.1.6
+    # via gradio
 jiter==0.11.1
+    # via openai
+joblib==1.5.2
+    # via scikit-learn
 jsonpatch==1.33
+    # via langchain-core
 jsonpointer==3.0.0
-jupyter-client==8.6.3
-jupyter-core==5.9.1
+    # via jsonpatch
+jsonschema==4.25.1
+    # via mcp
+jsonschema-specifications==2025.9.1
+    # via jsonschema
 langchain==1.0.2
+    # via hier-rag (pyproject.toml)
 langchain-classic==1.0.0
+    # via langchain-community
 langchain-community==0.4.1
-langchain-core==1.0.1
+    # via hier-rag (pyproject.toml)
+langchain-core==1.0.2
+    # via
+    #   langchain
+    #   langchain-classic
+    #   langchain-community
+    #   langchain-google-genai
+    #   langchain-milvus
+    #   langchain-openai
+    #   langchain-text-splitters
+    #   langgraph
+    #   langgraph-checkpoint
+    #   langgraph-prebuilt
 langchain-google-genai==3.0.0
+    # via langchain
 langchain-milvus==0.2.2
+    # via hier-rag (pyproject.toml)
 langchain-openai==1.0.1
+    # via langchain
 langchain-text-splitters==1.0.0
-langgraph==1.0.1
+    # via
+    #   hier-rag (pyproject.toml)
+    #   langchain-classic
+langgraph==1.0.2
+    # via langchain
 langgraph-checkpoint==3.0.0
-langgraph-prebuilt==1.0.1
+    # via
+    #   langgraph
+    #   langgraph-prebuilt
+langgraph-prebuilt==1.0.2
+    # via langgraph
 langgraph-sdk==0.2.9
-langsmith==0.4.38
+    # via langgraph
+langsmith==0.4.39
+    # via
+    #   langchain-classic
+    #   langchain-community
+    #   langchain-core
 markdown-it-py==4.0.0
+    # via rich
 markupsafe==3.0.3
+    # via
+    #   gradio
+    #   jinja2
 marshmallow==3.26.1
-matplotlib-inline==0.2.1
+    # via dataclasses-json
+mcp==1.10.1
+    # via gradio
 mdurl==0.1.2
+    # via markdown-it-py
 milvus-lite==2.5.1
+    # via hier-rag (pyproject.toml)
 multidict==6.7.0
+    # via
+    #   aiohttp
+    #   yarl
 mypy-extensions==1.1.0
-nest-asyncio==1.6.0
+    # via typing-inspect
 numpy==2.3.4
+    # via
+    #   gradio
+    #   langchain-community
+    #   pandas
+    #   rank-bm25
+    #   scikit-learn
+    #   scipy
 openai==2.6.1
+    # via langchain-openai
 orjson==3.11.4
+    # via
+    #   gradio
+    #   langgraph-sdk
+    #   langsmith
+    #   pymilvus
 ormsgpack==1.11.0
+    # via langgraph-checkpoint
 packaging==25.0
+    # via
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   langchain-core
+    #   langsmith
+    #   marshmallow
 pandas==2.3.3
-parso==0.8.5
+    # via
+    #   gradio
+    #   pymilvus
 pdfminer-six==20250506
-pexpect==4.9.0
+    # via hier-rag (pyproject.toml)
 pillow==11.3.0
-platformdirs==4.5.0
-pluggy==1.6.0
-prompt-toolkit==3.0.52
+    # via gradio
 propcache==0.4.1
+    # via
+    #   aiohttp
+    #   yarl
 proto-plus==1.26.1
+    # via
+    #   google-ai-generativelanguage
+    #   google-api-core
 protobuf==6.33.0
-psutil==7.1.2
-ptyprocess==0.7.0
-pure-eval==0.2.3
+    # via
+    #   google-ai-generativelanguage
+    #   google-api-core
+    #   googleapis-common-protos
+    #   grpcio-status
+    #   proto-plus
+    #   pymilvus
 pyasn1==0.6.1
+    # via
+    #   pyasn1-modules
+    #   rsa
 pyasn1-modules==0.4.2
+    # via google-auth
 pycparser==2.23
+    # via cffi
 pydantic==2.11.10
+    # via
+    #   fastapi
+    #   gradio
+    #   langchain
+    #   langchain-classic
+    #   langchain-core
+    #   langchain-google-genai
+    #   langgraph
+    #   langsmith
+    #   mcp
+    #   openai
+    #   pydantic-settings
 pydantic-core==2.33.2
+    # via pydantic
 pydantic-settings==2.11.0
+    # via
+    #   langchain-community
+    #   mcp
 pydub==0.25.1
+    # via gradio
 pygments==2.19.2
-pymilvus==2.6.2
-pytest==8.4.2
+    # via rich
+pymilvus==2.6.3
+    # via langchain-milvus
 python-dateutil==2.9.0.post0
+    # via pandas
 python-dotenv==1.2.1
+    # via
+    #   pydantic-settings
+    #   pymilvus
 python-multipart==0.0.20
+    # via
+    #   gradio
+    #   mcp
 pytz==2025.2
+    # via pandas
 pyyaml==6.0.3
-pyzmq==27.1.0
+    # via
+    #   gradio
+    #   huggingface-hub
+    #   langchain-classic
+    #   langchain-community
+    #   langchain-core
 rank-bm25==0.2.2
+    # via hier-rag (pyproject.toml)
+referencing==0.37.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
 regex==2025.10.23
+    # via tiktoken
 requests==2.32.5
+    # via
+    #   google-api-core
+    #   langchain-classic
+    #   langchain-community
+    #   langsmith
+    #   requests-toolbelt
+    #   tiktoken
 requests-toolbelt==1.0.0
+    # via langsmith
 rich==14.2.0
+    # via typer
+rpds-py==0.28.0
+    # via
+    #   jsonschema
+    #   referencing
 rsa==4.9.1
-ruff==0.14.2
+    # via google-auth
+ruff==0.14.3
+    # via gradio
 safehttpx==0.1.7
+    # via gradio
+scikit-learn==1.7.2
+    # via hier-rag (pyproject.toml)
+scipy==1.16.3
+    # via scikit-learn
 semantic-version==2.10.0
+    # via gradio
 setuptools==80.9.0
+    # via pymilvus
 shellingham==1.5.4
+    # via
+    #   huggingface-hub
+    #   typer
 six==1.17.0
+    # via python-dateutil
 sniffio==1.3.1
+    # via
+    #   anyio
+    #   openai
 sqlalchemy==2.0.44
-stack-data==0.6.3
-starlette==0.48.0
+    # via
+    #   langchain-classic
+    #   langchain-community
+sse-starlette==3.0.3
+    # via mcp
+starlette==0.49.3
+    # via
+    #   fastapi
+    #   gradio
+    #   mcp
 tenacity==9.1.2
+    # via
+    #   langchain-community
+    #   langchain-core
+threadpoolctl==3.6.0
+    # via scikit-learn
 tiktoken==0.12.0
+    # via langchain-openai
 tomlkit==0.13.3
-tornado==6.5.2
+    # via gradio
 tqdm==4.67.1
-traitlets==5.14.3
+    # via
+    #   huggingface-hub
+    #   milvus-lite
+    #   openai
 typer==0.20.0
+    # via gradio
 typer-slim==0.20.0
+    # via huggingface-hub
 typing-extensions==4.15.0
+    # via
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   grpcio
+    #   huggingface-hub
+    #   langchain-core
+    #   openai
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typer
+    #   typer-slim
+    #   typing-inspect
+    #   typing-inspection
 typing-inspect==0.9.0
+    # via dataclasses-json
 typing-inspection==0.4.2
+    # via
+    #   pydantic
+    #   pydantic-settings
 tzdata==2025.2
-ujson==5.11.0
-urllib3==2.3.0
+    # via pandas
+urllib3==2.5.0
+    # via requests
 uvicorn==0.38.0
-wcwidth==0.2.14
+    # via
+    #   gradio
+    #   mcp
 websockets==15.0.1
+    # via gradio-client
 xxhash==3.6.0
+    # via langgraph
 yarl==1.22.0
+    # via aiohttp
 zstandard==0.25.0
+    # via langsmith

app.py → src/app.py

@@ -1,18 +1,25 @@
 import gradio as gr
 import time
-from pathlib import Path
-from core.ingest import ingest
-from core.retrieval import generate, retrieval
-from core.index import MetaData
 import yaml
+import sys
+from pathlib import Path
+from dataclasses import asdict
+
+# Ensure project root is on sys.path when running this module as a script.
+_project_root = Path(__file__).resolve().parents[1]
+if str(_project_root) not in sys.path:
+    sys.path.insert(0, str(_project_root))
+
 
-# Import evaluation functions
-from core.eval import (
+from src.core.ingest import ingest
+from src.core.retrieval import generate, retrieval
+from src.core.index import MetaData
+from src.core.synthetic_data import EVAL_QUERIES, SYNTHETIC_DOCUMENTS
+from src.core.eval import (
     run_full_evaluation, 
     save_results, 
     generate_summary_report,
     setup_test_data,
-    EVAL_QUERIES
 )
 
 
@@ -183,12 +190,12 @@ def setup_synthetic_data(collections):
     
     try:
         docs_length = setup_test_data(collections)
-        return f"✅ Successfully ingested {docs_length} synthetic test data for each: {', '.join(collections)}"
+        return f"✅ Successfully ingested {docs_length} synthetic test data for: {', '.join(collections)}"
     except Exception as e:
         return f"❌ Error setting up test data: {str(e)}"
 
 
-def run_evaluation_batch(collections, output_dir):
+def run_evaluation_batch(collections, output_dir, progress=gr.Progress(track_tqdm=True)):
     """Run full batch evaluation"""
     if not collections:
         return (
@@ -248,11 +255,6 @@ def run_evaluation_batch(collections, output_dir):
             f"Error: {str(e)}"
         )
 
-def get_predefined_queries_list():
-    """Get list of predefined queries for dropdown"""
-    return [""] + [f"{i}: {q.model_dump()}" for i, q in enumerate(EVAL_QUERIES)]
-
-
 # --- Static choices (not from YAML) ---
 LANG_CHOICES = ["en", "ja"]
 DOC_TYPE_CHOICES = [None, "policy", "manual", "faq"]
@@ -423,12 +425,12 @@ with gr.Blocks(theme=gr.themes.Soft(), title="RAG Evaluation System") as demo:
     with gr.Tab("🧪 Evaluation"):
        
     
-        gr.Markdown("""
+        gr.Markdown(f"""
         ### Run Complete Evaluation
         
         This will:
-        1. Initial ingest synthetic test data (60 documents)
-        2. Run 15 predefined evaluation queries
+        1. Initial ingest synthetic test data ({sum(len(docs) for docs in SYNTHETIC_DOCUMENTS.values())} documents)
+        2. Run {len(EVAL_QUERIES)} predefined evaluation queries
         3. Generate comprehensive reports (CSV, JSON, Markdown)
         4. Compare Base RAG vs Hierarchical RAG
         """)
@@ -447,6 +449,11 @@ with gr.Blocks(theme=gr.themes.Soft(), title="RAG Evaluation System") as demo:
                     value="reports",
                     info="Directory where evaluation reports will be saved"
                 )
+
+                with gr.Accordion("SYNTHETIC_DOCUMENTS", open=False):
+                    gr.JSON(value=SYNTHETIC_DOCUMENTS)
+                with gr.Accordion("EVAL_QUERIES", open=False):
+                    gr.JSON(value=[asdict(q) for q in EVAL_QUERIES])
                 
                 with gr.Row():
                     setup_data_btn = gr.Button(
@@ -501,7 +508,8 @@ with gr.Blocks(theme=gr.themes.Soft(), title="RAG Evaluation System") as demo:
                 csv_download,
                 json_download,
                 eval_summary_md
-            ]
+            ],
+            show_progress="full"
         )
     
     # --- Event Handlers ---
@@ -537,4 +545,4 @@ with gr.Blocks(theme=gr.themes.Soft(), title="RAG Evaluation System") as demo:
 
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(mcp_server=True)

{core → src/core}/eval.py

@@ -7,6 +7,8 @@ import json
 import csv
 import time
 import uuid
+from tqdm import tqdm
+from random import shuffle
 from pathlib import Path
 from typing import List, Dict
 from datetime import datetime
@@ -190,27 +192,19 @@ def run_full_evaluation(
     
     # Filter queries by requested collections
     queries_to_eval = [q for q in EVAL_QUERIES if q.collection in collections]
-    
+    shuffle(queries_to_eval)
     print(f"\n{'='*70}")
     print(f"Starting Evaluation: {len(queries_to_eval)} queries across {len(collections)} collections")
     print(f"{'='*70}\n")
     
-    for i, eval_query in enumerate(queries_to_eval, 1):
-        print(f"[{i}/{len(queries_to_eval)}] Evaluating: {eval_query.description}")
-        print(f"  Collection: {eval_query.collection}")
-        print(f"  Query: {eval_query.query[:60]}...")
-        
+    for eval_query in tqdm(queries_to_eval, desc="Running evaluation queries"):
         # Evaluate with base RAG
-        print("  - Running Base RAG...")
         base_result = evaluate_single_query(eval_query, "base")
         all_results["base"].append(base_result)
         
         # Evaluate with hierarchical RAG
-        print("  - Running Hierarchical RAG...")
         hier_result = evaluate_single_query(eval_query, "hierarchical")
         all_results["hierarchical"].append(hier_result)
-        
-        print(f"  ✓ Complete (Base: {base_result.total_latency_ms:.0f}ms, Hier: {hier_result.total_latency_ms:.0f}ms)\n")
     
     return all_results
 
@@ -408,8 +402,10 @@ def generate_summary_report(results: Dict[str, List[EvalResult]], output_dir: st
         f.write("## Detailed Query Results\n\n")
         
         # Sample queries with comparison
-        for i, (base_r, hier_r) in enumerate(zip(base_results[:5], hier_results[:5]), 1):
+        for i, (base_r, hier_r) in enumerate(zip(base_results[:20], hier_results[:20]), 1):
             f.write(f"### Query {i}: {base_r.query}\n\n")
+            f.write(f"### Base Response {i}:\n{base_r.generated_answer}\n\n")
+            f.write(f"### Hier Response {i}:\n{hier_r.generated_answer}\n\n")
             f.write(f"**Collection:** {base_r.collection}\n\n")
             
             f.write("| Aspect | Base RAG | Hierarchical RAG |\n")
@@ -431,7 +427,7 @@ def setup_test_data(collections: List[str] = None):
     print("\n" + "="*70)
     print("Setting up test data for evaluation")
     print("="*70 + "\n")
-    
+    tot_docs = 0
     for collection_name in collections:
         if collection_name not in SYNTHETIC_DOCUMENTS:
             print(f"⚠️  No synthetic data available for '{collection_name}', skipping...")
@@ -450,11 +446,11 @@ def setup_test_data(collections: List[str] = None):
         vectorstore = get_vectorstore(collection_name)
         ids = [str(uuid.uuid4()) for _ in range(len(documents))]
         vectorstore.add_documents(documents, ids=ids)
-
+        tot_docs += len(documents)
         print(f"✓ Completed '{collection_name}' collection")
     
     print("\n" + "="*70)
     print("Test data setup complete!")
     print("="*70 + "\n")
 
-    return len(documents)
+    return tot_docs

{core → src/core}/index.py

@@ -23,7 +23,7 @@ class MetaData(BaseModel):
 model = ChatOpenAI(model="gpt-5-nano")
 emb_model = OpenAIEmbeddings(model="text-embedding-3-small", dimensions=1536)
 
-MILVUS_URI = "./rag_task.db"
+MILVUS_URI = "./data/rag_task.db"
 
 
 def get_vectorstore(collection_name: str) -> Milvus:

{core → src/core}/ingest.py

@@ -1,4 +1,4 @@
-from langchain_community.document_loaders import PDFMinerLoader
+from langchain_community.document_loaders import PDFMinerLoader,TextLoader
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_core.documents import Document
 from langchain_openai import ChatOpenAI
@@ -17,7 +17,10 @@ model = ChatOpenAI(model="gpt-5-nano")
 def ingest(file_paths: List[str], collection_name: str, metadata: MetaData):
     documents: list[Document] = []
     for file_path in file_paths:
-        docs = PDFMinerLoader(file_path).load()
+        if file_path.endswith(".txt"):
+            docs = TextLoader(file_path, encoding="utf-8").load()
+        elif file_path.endswith(".pdf"):
+            docs = PDFMinerLoader(file_path).load()
         documents.extend(docs)
         for doc in docs:
             doc.metadata["source"] = file_path.split("/")[-1]
@@ -39,7 +42,6 @@ def ingest(file_paths: List[str], collection_name: str, metadata: MetaData):
                 "doc_id": doc_id,
                 "chunk_id": str(uuid.uuid4()),
                 "source_name": chunk.metadata["source"],
-                "total_pages": chunk.metadata["total_pages"],
                 "start_index": chunk.metadata["start_index"],
                 **metadata.model_dump(),
             },

src/core/rag.ipynb

@@ -0,0 +1,244 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "a57aab57",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from langchain_google_genai import GoogleGenerativeAIEmbeddings, ChatGoogleGenerativeAI\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain_openai.embeddings import OpenAIEmbeddings\n",
+    "from langchain_milvus import Milvus, BM25BuiltInFunction\n",
+    "from typing import Literal, Optional\n",
+    "from pydantic import BaseModel\n",
+    "from dotenv import load_dotenv, find_dotenv\n",
+    "\n",
+    "find_dotenv()\n",
+    "load_dotenv()\n",
+    "\n",
+    "\n",
+    "class MetaData(BaseModel):\n",
+    "    language: Literal[\"ja\", \"en\"]\n",
+    "    domain: Optional[str] = None\n",
+    "    section: Optional[str] = None\n",
+    "    topic: Optional[str] = None\n",
+    "    doc_type: Optional[Literal[\"policy\", \"manual\", \"faq\"]] = None\n",
+    "\n",
+    "\n",
+    "# model = ChatGoogleGenerativeAI(model=\"models/gemini-2.5-flash-lite\")\n",
+    "# emb_model = GoogleGenerativeAIEmbeddings(model=\"models/gemini-embedding-001\", output_dimensionality=1536)\n",
+    "model = ChatOpenAI(model=\"gpt-5-nano\")\n",
+    "emb_model = OpenAIEmbeddings(model=\"text-embedding-3-small\", dimensions=1536)\n",
+    "\n",
+    "MILVUS_URI = \"./rag_task.db\"\n",
+    "\n",
+    "\n",
+    "def get_vectorstore(collection_name: str) -> Milvus:\n",
+    "    vectorstore = Milvus(\n",
+    "        embedding_function=emb_model,\n",
+    "        collection_name=collection_name,\n",
+    "        connection_args={\"uri\": MILVUS_URI},\n",
+    "        index_params={\"index_type\": \"FLAT\", \"metric_type\": \"L2\"},\n",
+    "    )\n",
+    "    # builtin_function=BM25BuiltInFunction(output_field_names=\"sparse\"),\n",
+    "    # text_field=\"text\",\n",
+    "    # vector_field=[\"dense\", \"sparse\"],\n",
+    "    print(f\"vectorstore successfully initialized for {collection_name}\")\n",
+    "    return vectorstore\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "db72701e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "\n",
+    "\n",
+    "def mask_pii(text: str) -> str:\n",
+    "    \"\"\"Mask Personally Identifiable Information\"\"\"\n",
+    "    # Email addresses\n",
+    "    text = re.sub(r'\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}\\b', '[EMAIL]', text)\n",
+    "    \n",
+    "    # Phone numbers\n",
+    "    text = re.sub(r'\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b', '[PHONE]', text)\n",
+    "    \n",
+    "    # Credit card numbers\n",
+    "    text = re.sub(r'\\b\\d{4}[- ]?\\d{4}[- ]?\\d{4}[- ]?\\d{4}\\b', '[CREDIT_CARD]', text)\n",
+    "    \n",
+    "    # Social Security Numbers\n",
+    "    text = re.sub(r'\\b\\d{3}-\\d{2}-\\d{4}\\b', '[SSN]', text)\n",
+    "    \n",
+    "    return text\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "f6037cfd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import PDFMinerLoader\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "from langchain_core.documents import Document\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from dotenv import load_dotenv, find_dotenv\n",
+    "from typing import List\n",
+    "import uuid\n",
+    "\n",
+    "\n",
+    "find_dotenv()\n",
+    "load_dotenv()\n",
+    "\n",
+    "model = ChatOpenAI(model=\"gpt-5-nano\")\n",
+    "\n",
+    "\n",
+    "def ingest(file_paths: List[str], collection_name: str, metadata: MetaData):\n",
+    "    documents: list[Document] = []\n",
+    "    for file_path in file_paths:\n",
+    "        docs = PDFMinerLoader(file_path).load()\n",
+    "        documents.extend(docs)\n",
+    "        for doc in docs:\n",
+    "            doc.metadata[\"source\"] = file_path.split(\"/\")[-1]\n",
+    "          \n",
+    "    print(f\"loaded {len(documents)} documents from {len(file_paths)} files.\")\n",
+    "    text_splitter = RecursiveCharacterTextSplitter(\n",
+    "        chunk_size=1200,  # chunk size (characters)\n",
+    "        chunk_overlap=200,  # chunk overlap (characters)\n",
+    "        add_start_index=True,  # track index in original document\n",
+    "    )\n",
+    "    chunks = text_splitter.split_documents(documents)\n",
+    "    print(f\"generated {len(chunks)} chunks.\")\n",
+    "\n",
+    "    doc_id = str(uuid.uuid4())\n",
+    "    docs = [\n",
+    "        Document(\n",
+    "            page_content=mask_pii(chunk.page_content),\n",
+    "            metadata={\n",
+    "                \"doc_id\": doc_id,\n",
+    "                \"chunk_id\": str(uuid.uuid4()),\n",
+    "                \"source_name\": chunk.metadata[\"source\"],\n",
+    "                \"total_pages\": chunk.metadata[\"total_pages\"],\n",
+    "                \"start_index\": chunk.metadata[\"start_index\"],\n",
+    "                **metadata.model_dump(),\n",
+    "            },\n",
+    "        )\n",
+    "        for chunk in chunks\n",
+    "    ]\n",
+    "\n",
+    "    vectorstore = get_vectorstore(collection_name)\n",
+    "    ids = [str(uuid.uuid4()) for _ in range(len(docs))]\n",
+    "    vectorstore.add_documents(docs, ids=ids)\n",
+    "    success_message = f\"Ingested {len(docs)} documents into {collection_name} index.\"\n",
+    "    print(success_message)\n",
+    "    return success_message\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "92a1751f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_core.documents import Document\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain_community.retrievers import BM25Retriever\n",
+    "from dotenv import load_dotenv, find_dotenv\n",
+    "from typing import List\n",
+    "\n",
+    "find_dotenv()\n",
+    "load_dotenv()\n",
+    "\n",
+    "model = ChatOpenAI(model=\"gpt-5-nano\")\n",
+    "\n",
+    "\n",
+    "def reranker(query: str, docs: List[Document]) -> List[Document]:\n",
+    "    print(f\"Retrieved {len(docs)} documents\")\n",
+    "    retriever = BM25Retriever.from_documents(docs)\n",
+    "    result = retriever.invoke(query)\n",
+    "    print(\"RERANKER Result: \", len(result), result[0])\n",
+    "    return result\n",
+    "\n",
+    "\n",
+    "def retrieval(\n",
+    "    query: str, collection_name: str, filter_data: MetaData\n",
+    ") -> List[tuple[Document, float]]:\n",
+    "    vectorstore = get_vectorstore(collection_name)\n",
+    "    print(\n",
+    "        f\"RETRIEVAL query: {query[:40]}, for {collection_name} collection, with filters: {filter_data}\"\n",
+    "    )\n",
+    "\n",
+    "    filters = [f'language == \"{filter_data.language}\"']\n",
+    "    if filter_data.doc_type:\n",
+    "        filters.append(f'doc_type == \"{filter_data.doc_type}\"')\n",
+    "    if filter_data.domain:\n",
+    "        filters.append(f'domain == \"{filter_data.domain}\"')\n",
+    "    if filter_data.section:\n",
+    "        filters.append(f'section == \"{filter_data.section}\"')\n",
+    "    if filter_data.topic:\n",
+    "        filters.append(f'topic == \"{filter_data.topic}\"')\n",
+    "\n",
+    "    expr = \" and \".join(filters) if filters else None\n",
+    "    try:\n",
+    "        results = vectorstore.similarity_search_with_relevance_scores(\n",
+    "            query, k=5, expr=expr\n",
+    "        )\n",
+    "    except ValueError as e:\n",
+    "        print(f\"Error in retrieval: {str(e)}\")\n",
+    "        return []\n",
+    "    docs = []\n",
+    "    for doc, score in results:\n",
+    "        doc.metadata[\"similarity_score\"] = score\n",
+    "        docs.append(doc)\n",
+    "    # docs = reranker(query, docs)\n",
+    "    print(\"RETRIEVED DOCS: \", len(docs))\n",
+    "    return docs\n",
+    "\n",
+    "\n",
+    "def generate(query: str, ctx_docs: List[Document]) -> str:\n",
+    "    context = \"\\n\".join([doc.page_content for doc in ctx_docs])\n",
+    "    prompt = f\"\"\"Answer shortly to the user question according to the given context. Only answer if the context is given to you.\n",
+    "    question: {query}\n",
+    "    context: {context}\n",
+    "\"\"\"\n",
+    "    output = model.invoke(prompt)\n",
+    "    return output.content\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4fb1e93f",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hier-rag",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{core → src/core}/retrieval.py

@@ -13,10 +13,12 @@ model = ChatOpenAI(model="gpt-5-nano")
 
 def reranker(query: str, docs: List[Document]) -> List[Document]:
     print(f"Retrieved {len(docs)} documents")
+    if len(docs) <= 1:
+        return docs
     retriever = BM25Retriever.from_documents(docs)
-    result = retriever.invoke(query)
-    print("RERANKER Result: ", len(result), result[0])
-    return result
+    docs = retriever.invoke(query)
+    print("RERANKER Result: ", len(docs))
+    return docs
 
 
 def retrieval(

{core → src/core}/utils.py

@@ -7,7 +7,7 @@ def mask_pii(text: str) -> str:
     text = re.sub(r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b', '[EMAIL]', text)
     
     # Phone numbers
-    text = re.sub(r'\b\d{3}[-.]?\d{3}[-.]?\d{4}\b', '[PHONE]', text)
+    text = re.sub(r'\b(?:\d{3}[-.]?\d{4}|\d{3}[-.]?\d{3}[-.]?\d{4})\b', '[PHONE]', text)
     
     # Credit card numbers
     text = re.sub(r'\b\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}\b', '[CREDIT_CARD]', text)