Upload 11 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/1706.03762v7.pdf filter=lfs diff=lfs merge=lfs -text
+data/recommendations-for-regulating-ai.pdf filter=lfs diff=lfs merge=lfs -text
+data/Usage[[:space:]]policies[[:space:]]_[[:space:]]OpenAI.pdf filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,78 @@
+import streamlit as st
+import os
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger("APP_ENTRY")
+
+logger.info("🚀 app.py module loaded. Streamlit starting up...")
+
+st.set_page_config(page_title="Gemini Research Assistant", layout="wide")
+st.title("💎 Agentic RAG: Gemini 2.0 Research Assistant")
+
+# --- AUTO-INGESTION SEQUENCE ---
+# This ensures the vector DB exists before the agent tries to load it.
+# --- CONFIGURATION ---
+DB_PATH = "./chroma_db"
+DATA_PATH = "./data"
+
+@st.cache_resource(show_spinner=False)
+def initialize_knowledge_base():
+    """Checks and builds the vector database if missing."""
+    if not os.path.exists(DB_PATH) or not os.listdir(DB_PATH):
+        logger.info("⚠️ VectorDB not found. Checking for PDF data...")
+        if os.path.exists(DATA_PATH) and any(f.endswith('.pdf') for f in os.listdir(DATA_PATH)):
+            logger.info("📄 Data found. Starting ingestion process...")
+            # We use a placeholder to show progress since st.spinner isn't thread-safe in early startup sometimes
+            status_placeholder = st.empty()
+            status_placeholder.info("🧠 Initializing Knowledge Base... Check Logs for progress.")
+            
+            from src.processor import build_index
+            try:
+                build_index(DATA_PATH, DB_PATH)
+                status_placeholder.success("✅ Knowledge Base Built! Refreshing...")
+                logger.info("✅ Ingestion complete.")
+                status_placeholder.empty()
+            except Exception as e:
+                logger.error(f"❌ Ingestion FAILED: {e}")
+                status_placeholder.error(f"Failed to build index: {e}")
+                raise e
+        else:
+            logger.warning("No data found in 'data' directory.")
+            st.warning("⚠️ No data found! Please add PDFs to the 'data' folder to use Local Research.")
+    else:
+        logger.info("✅ VectorDB exists. Skipping ingestion.")
+
+# Run the initialization
+initialize_knowledge_base()
+
+# Lazy import agent AFTER DB check to prevent "Table not found" errors
+logger.info("🤖 Loading Agent Logic...")
+from src.agent import app as agent_app
+logger.info("✅ Agent loaded. Ready to serve.")
+
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display history
+for msg in st.session_state.messages:
+    with st.chat_message(msg["role"]):
+        st.markdown(msg["content"])
+
+# Chat input
+if prompt := st.chat_input("Ask about internal docs or latest tech..."):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    with st.chat_message("assistant"):
+        inputs = {"messages": [("user", prompt)]}
+        config = {"configurable": {"thread_id": "1"}}
+        
+        # Execute LangGraph brain
+        response = agent_app.invoke(inputs, config=config)
+        answer = response["messages"][-1].content
+        
+        st.markdown(answer)
+        st.session_state.messages.append({"role": "assistant", "content": answer})

data/1706.03762v7.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdfaa68d8984f0dc02beaca527b76f207d99b666d31d1da728ee0728182df697
+size 2215244

data/Usage policies _ OpenAI.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65b3fd64e61ca4bdeac41fea6c44d8a927a3e16129b94af7118196848f0c7c6f
+size 145434

data/recommendations-for-regulating-ai.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abbefed52379ac6bd793071bc603e174b233718a4fc8ad32ac304edae1e39425
+size 316312

ingest.py

@@ -0,0 +1,28 @@
+import os
+import shutil
+from src.processor import build_index
+
+DATA_DIR = "./data"
+DB_DIR = "./chroma_db"
+
+def main():
+    print(f"Checking for data in {DATA_DIR}...")
+    if not os.path.exists(DATA_DIR):
+        print(f"Create a '{DATA_DIR}' directory and put your PDFs there.")
+        return
+
+    if not any(f.endswith(".pdf") for f in os.listdir(DATA_DIR)):
+        print("No PDF files found in data directory.")
+        return
+
+    print("Building Vector Database... (This may take a while for large docs)")
+    
+    # Optional: Clear old DB if you want a fresh start every time
+    # if os.path.exists(DB_DIR):
+    #     shutil.rmtree(DB_DIR)
+
+    vectorstore = build_index(DATA_DIR, DB_DIR)
+    print(f"Success! Database built at {DB_DIR}")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -1,3 +1,15 @@
-altair
-pandas
-streamlit
+langchain
+langchain-google-genai
+langchain-chroma
+langchain-community
+langchain-text-splitters
+langgraph
+streamlit
+chromadb
+pypdf
+ftfy
+unidecode
+ragas
+datasets
+duckduckgo-search
+python-dotenv

src/agent.py

@@ -0,0 +1,68 @@
+from typing import Annotated, List, TypedDict
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langgraph.graph import StateGraph, START, END
+from langgraph.prebuilt import ToolNode
+from src.tools import tools
+
+# The Local Knowledge Registry (Update this whenever you add new data types)
+# Manual as of now
+LOCAL_MANIFEST = {
+    "topics": ["HR Policies", "Project X Design Docs", "Q3 Financials", "Employee Handbook"],
+    "date_range": "Documents updated as of Dec 2024",
+    "domain": "Internal Corporate Knowledge"
+}
+
+SYSTEM_PROMPT = f"""
+You are an expert Research Assistant. You have access to:
+1. INTERNAL DATA: {LOCAL_MANIFEST['topics']}. (Use 'local_research_tool')
+2. EXTERNAL DATA: The entire internet via duckduckgosearch. (Use 'web_search_tool')
+
+GUIDELINES:
+- Given the user's technical question and the fact that our internal documents are insufficient, generate a generic search query for the internet that does NOT include any proprietary names or internal details.
+- If a query is about {LOCAL_MANIFEST['topics']}, try LOCAL first.
+- If a query is TECHNICAL (e.g., PyTorch, Python APIs) or REAL-TIME, go to WEB immediately.
+- If the query is ambiguous, try LOCAL first, then fallback to WEB if the results are empty or low confidence.
+"""
+
+
+class AgentState(TypedDict):
+    """MNC Agent state with intent tracking and sufficiency grading."""
+    messages: Annotated[List, "Chat history"]
+    intent: str
+    is_sufficient: bool
+
+# Brain: Gemini 2.0 Flash (high-speed agentic reasoning)
+llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0)
+llm_with_tools = llm.bind_tools(tools)
+
+def router(state: AgentState):
+    """Classifies user intent to prioritize retrieval paths."""
+    query = state['messages'][-1].content
+    prompt = f"Categorize intent: TECHNICAL (API/Docs), INTERNAL (Proprietary), or REALTIME. Query: {query}"
+    response = llm.invoke(prompt)
+    intent = "TECHNICAL" if any(x in response.content.upper() for x in ["TECHNICAL", "REALTIME"]) else "INTERNAL"
+    return {"intent": intent}
+
+def call_model(state: AgentState):
+    """Invokes Gemini with tools based on intent and history."""
+    return {"messages": [llm_with_tools.invoke(state['messages'])]}
+
+# Orchestration Graph
+workflow = StateGraph(AgentState)
+
+workflow.add_node("router", router)
+workflow.add_node("llm", call_model)
+workflow.add_node("tools", ToolNode(tools))
+
+workflow.add_edge(START, "router")
+workflow.add_edge("router", "llm")
+
+# Self-Correction Loop
+def should_continue(state: AgentState):
+    last_msg = state["messages"][-1]
+    return "tools" if last_msg.tool_calls else END
+
+workflow.add_conditional_edges("llm", should_continue)
+workflow.add_edge("tools", "llm")
+
+app = workflow.compile()

src/eval.py

@@ -0,0 +1,26 @@
+from datasets import Dataset
+from ragas import evaluate
+from ragas.llms import llm_factory
+from ragas.metrics.collections import Faithfulness, ResponseRelevancy, ContextPrecision
+from src.agent import app
+
+# Ragas 2025: Experiment-based factory
+judge_llm = llm_factory("gemini-2.0-flash")
+
+def evaluate_agent(questions: list, references: list):
+    """MNC-grade verification of RAG pipeline quality."""
+    results = []
+    for q, r in zip(questions, references):
+        output = app.invoke({"messages": [("user", q)]})
+        results.append({
+            "user_input": q,
+            "response": output["messages"][-1].content,
+            "retrieved_contexts": [m.content for m in output["messages"] if hasattr(m, "tool_call_id")],
+            "reference": r
+        })
+    
+    dataset = Dataset.from_list(results)
+    metrics = [Faithfulness(), ResponseRelevancy(), ContextPrecision()]
+    
+    # Evaluate with Gemini judge
+    return evaluate(dataset=dataset, metrics=metrics, llm=judge_llm)

src/processor.py

@@ -0,0 +1,76 @@
+import re
+import unicodedata
+import hashlib
+from pathlib import Path
+import ftfy
+import unidecode
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_chroma import Chroma
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import DirectoryLoader, PyPDFLoader
+
+import logging
+
+# Configure logging to show up in Docker logs
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+def clean_text(text: str) -> str:
+    """MNC-grade scrubbing for structural and encoding noise."""
+    # 1. Structural Scrubbing
+    text = re.sub(r'Page\s+\d+\s+of\s+\d+', '', text, flags=re.IGNORECASE)
+    text = re.sub(r'\b\d+\s*/\s*\d+\b', '', text)
+    text = re.sub(r'^\s*-\s*\d+\s*-\s*$', '', text, flags=re.MULTILINE)
+    text = re.sub(r'[-*_]{3,}', '', text)
+    
+    # 2. Encoding Repairs
+    text = ftfy.fix_text(text)
+    text = unidecode.unidecode(text)
+    text = unicodedata.normalize('NFKC', text)
+    
+    # 3. Whitespace Normalization
+    text = re.sub(r'[\t\xa0]', ' ', text)
+    text = re.sub(r'(?<=[a-z])\n(?=[a-z])', ' ', text) # Fix mid-sentence breaks
+    text = re.sub(r' +', ' ', text)
+    return text.strip()
+
+def build_index(data_dir: str, persist_dir: str):
+    """Processes messy data into a professional vector store."""
+    logger.info(f"Starting ingestion from: {data_dir}")
+    
+    loader = DirectoryLoader(data_dir, glob="**/*.pdf", loader_cls=PyPDFLoader)
+    raw_docs = loader.load()
+    logger.info(f"Loaded {len(raw_docs)} documents.")
+    
+    # Gemini 2025 standard embedding model
+    embeddings = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1200, chunk_overlap=150, add_start_index=True
+    )
+    
+    final_chunks = []
+    for i, doc in enumerate(raw_docs):
+        logger.info(f"Processing doc {i+1}/{len(raw_docs)}: {doc.metadata.get('source', 'unknown')}")
+        cleaned_content = clean_text(doc.page_content)
+        source_name = Path(doc.metadata.get("source", "unknown")).name
+        
+        # Metadata extraction for citations
+        metadata = {
+            "source": source_name,
+            "page": doc.metadata.get("page", 1),
+            "chunk_hash": hashlib.md5(cleaned_content.encode()).hexdigest()
+        }
+        
+        chunks = splitter.create_documents([cleaned_content], metadatas=[metadata])
+        final_chunks.extend(chunks)
+        
+    logger.info(f"Total chunks created: {len(final_chunks)}")
+    logger.info(f"Persisting to VectorDB at: {persist_dir}")
+    
+    vectorstore = Chroma.from_documents(
+        documents=final_chunks, 
+        embedding=embeddings, 
+        persist_directory=persist_dir
+    )
+    logger.info("VectorDB successfully built and persisted.")
+    return vectorstore

src/tools.py

@@ -0,0 +1,36 @@
+from typing import Literal
+from langchain_google_genai import GoogleGenerativeAIEmbeddings
+from langchain_chroma import Chroma
+from langchain_community.tools.duckduckgo_search import DuckDuckGoSearchResults
+from langchain.tools import tool
+
+# Persistent storage setup
+embeddings = GoogleGenerativeAIEmbeddings(model="models/text-embedding-004")
+vector_db = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
+
+@tool
+def local_research_tool(query: str, search_type: Literal["similarity", "mmr"] = "similarity"):
+    """
+    Searches the internal corporate knowledge base. 
+    Use 'similarity' for exact facts and 'mmr' for broad, diverse research.
+    """
+    retriever = vector_db.as_retriever(
+        search_type=search_type, 
+        search_kwargs={"k": 5, "fetch_k": 20, "lambda_mult": 0.5}
+    )
+    docs = retriever.invoke(query)
+    
+    # Formatted for model synthesis with citations
+    formatted = [
+        f"SOURCE: {d.metadata['source']} (Pg. {d.metadata['page']})\nCONTENT: {d.page_content}" 
+        for d in docs
+    ]
+    return "\n---\n".join(formatted)
+
+# Gemini-optimized web search fallback
+web_search_tool = DuckDuckGoSearchResults(
+    k=3,
+    description="Search the internet for real-time data, technical APIs (like PyTorch/LangChain), or news."
+)
+
+tools = [local_research_tool, web_search_tool]