add project

.gitattributes

@@ -0,0 +1,4 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.docx filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,10 @@
+.venv/
+.env
+__pycache__/
+*.pyc
+.streamlit/
+temp_docs/
+temp_faiss_index/
+debug_*.txt
+test_*.txt
+*.log

agents/graph.py

@@ -0,0 +1,171 @@
+"""
+LangGraph Workflow for Insurance RAG System.
+Implements deterministic, compliance-focused retrieval with specialized nodes.
+"""
+
+from langgraph.graph import StateGraph, END
+from agents.states import AgentState
+from agents.nodes import nodes
+
+
+def build_rag_workflow() -> StateGraph:
+    """
+    Builds the LangGraph workflow with the following flow:
+    
+    query_rewriter → query_classifier → entity_extractor → retrieval_router
+                              ↓
+                    [conditional routing by intent]
+                              ↓
+        ┌─────────────────────────────────────────────────────┐
+        │ list_plans: listing_agent → guardrail              │
+        │ plan_details: retriever → aggregator → retrieval → guardrail │
+        │ compare_plans: retriever → aggregator → comparison → guardrail │
+        │ recommendation: retriever → aggregator → advisory → guardrail │
+        │ general_query: retriever → aggregator → faq → guardrail │
+        └─────────────────────────────────────────────────────┘
+    """
+    
+    workflow = StateGraph(AgentState)
+    
+    # =========================================================================
+    # Add all nodes
+    # =========================================================================
+    
+    # Pre-processing nodes
+    workflow.add_node("query_rewriter", nodes.query_rewriter_node)
+    workflow.add_node("query_classifier", nodes.query_classifier_node)
+    workflow.add_node("entity_extractor", nodes.entity_extractor_node)
+    workflow.add_node("retrieval_router", nodes.retrieval_router_node)
+    
+    # Retrieval nodes
+    workflow.add_node("retriever", nodes.retriever_node)
+    workflow.add_node("plan_aggregator", nodes.plan_aggregator_node)
+    
+    # Agent nodes
+    workflow.add_node("listing_agent", nodes.listing_agent)
+    workflow.add_node("retrieval_agent", nodes.retrieval_agent)
+    workflow.add_node("comparison_agent", nodes.comparison_agent)
+    workflow.add_node("advisory_agent", nodes.advisory_agent)
+    workflow.add_node("faq_agent", nodes.faq_agent)
+    
+    # Post-processing
+    workflow.add_node("guardrail", nodes.guardrail_node)
+    
+    # =========================================================================
+    # Define edges
+    # =========================================================================
+    
+    # Entry point
+    workflow.set_entry_point("query_rewriter")
+    
+    # Linear pre-processing chain
+    workflow.add_edge("query_rewriter", "query_classifier")
+    workflow.add_edge("query_classifier", "entity_extractor")
+    workflow.add_edge("entity_extractor", "retrieval_router")
+    
+    # Conditional routing based on intent
+    def route_by_intent(state: AgentState) -> str:
+        """Route to appropriate handler based on classified intent."""
+        intent = state.get("intent", "plan_details")
+        
+        if intent == "list_plans":
+            # Listing doesn't need retrieval, goes direct to listing agent
+            return "listing_agent"
+        else:
+            # All other intents go through retrieval first
+            return "retriever"
+    
+    workflow.add_conditional_edges(
+        "retrieval_router",
+        route_by_intent,
+        {
+            "listing_agent": "listing_agent",
+            "retriever": "retriever"
+        }
+    )
+    
+    # Listing agent goes directly to guardrail
+    workflow.add_edge("listing_agent", "guardrail")
+    
+    # Retriever always goes to aggregator
+    workflow.add_edge("retriever", "plan_aggregator")
+    
+    # Aggregator routes to appropriate agent based on intent
+    def route_to_agent(state: AgentState) -> str:
+        """Route from aggregator to the appropriate agent."""
+        intent = state.get("intent", "plan_details")
+        
+        route_map = {
+            "plan_details": "retrieval_agent",
+            "compare_plans": "comparison_agent",
+            "recommendation": "advisory_agent",
+            "general_query": "faq_agent"
+        }
+        
+        return route_map.get(intent, "retrieval_agent")
+    
+    workflow.add_conditional_edges(
+        "plan_aggregator",
+        route_to_agent,
+        {
+            "retrieval_agent": "retrieval_agent",
+            "comparison_agent": "comparison_agent",
+            "advisory_agent": "advisory_agent",
+            "faq_agent": "faq_agent"
+        }
+    )
+    
+    # All agents end at guardrail
+    workflow.add_edge("retrieval_agent", "guardrail")
+    workflow.add_edge("comparison_agent", "guardrail")
+    workflow.add_edge("advisory_agent", "guardrail")
+    workflow.add_edge("faq_agent", "guardrail")
+    
+    # Guardrail ends the workflow
+    workflow.add_edge("guardrail", END)
+    
+    return workflow
+
+
+# Build and compile the workflow
+workflow = build_rag_workflow()
+app = workflow.compile()
+
+
+if __name__ == "__main__":
+    # Test the graph
+    print("Graph compiled successfully!")
+    
+    # Test cases
+    test_queries = [
+        "List all term plans from Tata AIA",
+        "Explain the TATA AIA Smart Value Income plan",
+        "Compare Tata AIA vs Edelweiss term plans",
+        "Suggest a plan for 30-year-old non-smoker with 1Cr cover"
+    ]
+    
+    for query in test_queries:
+        print(f"\n{'='*60}")
+        print(f"Query: {query}")
+        print('='*60)
+        
+        initial_state = {
+            "input": query,
+            "chat_history": [],
+            "intent": "",
+            "extracted_entities": {},
+            "metadata_filters": {},
+            "retrieval_strategy": "",
+            "context": [],
+            "retrieved_chunks": {},
+            "reasoning_output": "",
+            "answer": "",
+            "next_step": ""
+        }
+        
+        try:
+            result = app.invoke(initial_state)
+            print(f"Intent: {result.get('intent')}")
+            print(f"Answer: {result.get('answer', '')[:500]}...")
+        except Exception as e:
+            print(f"Error: {e}")

agents/nodes.py

@@ -0,0 +1,884 @@
+import re
+from typing import Dict, List, Any, Optional
+from collections import defaultdict
+from agents.states import AgentState, ExtractedEntities
+from rag.retriever import RAGRetriever
+from langchain_core.messages import HumanMessage, SystemMessage
+from models.llm import LLMFactory
+
+
+# Compliance disclaimer to append to all answers
+COMPLIANCE_DISCLAIMER = (
+    "\n\n---\n"
+)
+
+# Prompting rules for all agents
+COMPLIANCE_RULES = """
+CRITICAL RULES:
+- ❌ NO invented plan names - only use plans from the provided context
+- ❌ NO assumptions beyond documents - if info is missing, say so explicitly
+- ❌ NO meta-commentary. DO NOT mention "the provided context", "the documents", "the text", or "internal state".
+- ✅ CIS overrides brochure for: exclusions, charges, conditions
+- ✅ Use structured output (markdown tables) for comparisons
+- ✅ Simple, clear language for end users
+- ✅ Provide "OUTPUT ONLY" - start answering the user's question directly.
+"""
+
+
+class AgentNodes:
+    """
+    Enhanced LangGraph nodes implementing the full RAG specification.
+    """
+    
+    def __init__(self):
+        self.retriever = None
+
+    def _get_retriever(self) -> Optional[RAGRetriever]:
+        """Lazy initialization of retriever."""
+        if not self.retriever:
+            try:
+                self.retriever = RAGRetriever()
+            except Exception:
+                return None
+        return self.retriever
+
+    def reload_retriever(self):
+        """Triggers a reload of the retriever's index."""
+        retriever = self._get_retriever()
+        if retriever:
+            retriever.reload()
+
+    # =========================================================================
+    # NODE 1: Query Rewriter
+    # =========================================================================
+    def query_rewriter_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Rewrites query to be self-contained based on chat history.
+        Resolves pronouns and references.
+        """
+        llm = LLMFactory.get_llm("small")
+        query = state["input"]
+        history = state.get("chat_history", [])
+        
+        if not history:
+            return {"input": query}
+            
+        system_prompt = (
+            "You are a query rewriter for an insurance RAG system. "
+            "Your task is to rewrite the latest question to be self-contained.\n\n"
+            "RULES:\n"
+            "1. ALWAYS resolve pronouns (it, they, these) or vague terms (the plan, previous one) using the previous context.\n"
+            "2. If the user asks a follow-up about 'it' or 'the plan', replace it with the specific plan name mentioned last.\n"
+            "3. If the user asks 'is it good for me' or similar, rewrite it to '[Plan Name] recommendation for [user details if any]'.\n"
+            "4. If the query is already very specific and names a plan, keep it mostly as-is but ensure insurer names are present.\n"
+            "5. Do NOT cross-pollinate unrelated queries. If the user switches topics completely, ignore the history.\n"
+            "6. NEVER return a conversational response, suggestion, or question. If you cannot resolve a reference, return the original 'Latest' query as is.\n"
+            "7. Return ONLY the rewritten query text."
+        )
+        
+        history_str = "\n".join([f"- {h}" for h in history[-5:]])  # Last 5 turns
+        prompt = f"History:\n{history_str}\n\nLatest: {query}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        rewritten = getattr(response, 'content', str(response)).strip()
+        
+        return {"input": rewritten}
+
+    # =========================================================================
+    # NODE 2: Query Classifier
+    # =========================================================================
+    def query_classifier_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Classifies user intent into:
+        - list_plans: User wants to see available plans
+        - plan_details: User asks about a specific plan
+        - compare_plans: User wants to compare multiple plans
+        - recommendation: User seeks personalized advice
+        - general_query: General insurance questions
+        """
+        llm = LLMFactory.get_llm("small")
+        query = state["input"].lower()
+        
+        # Fast keyword-based classification first
+        if any(kw in query for kw in ["list", "which plans", "what plans", "all plans", "available plans", "show me plans"]):
+            return {"intent": "list_plans"}
+        
+        if any(kw in query for kw in ["compare", "vs", "versus", "difference between", "which is better"]):
+            return {"intent": "compare_plans"}
+        
+        if any(kw in query for kw in ["suggest", "recommend", "best for", "should i", "suitable for"]):
+            return {"intent": "recommendation"}
+        
+        # LLM-based classification for ambiguous cases
+        system_prompt = (
+            "Classify the user's insurance query into ONE of:\n"
+            "- 'plan_details': Asking about features, benefits, eligibility of a SPECIFIC plan\n"
+            "- 'list_plans': Wants to know WHICH plans are available\n"
+            "- 'compare_plans': Wants to COMPARE 2+ plans side-by-side\n"
+            "- 'recommendation': Seeks personalized advice based on their profile\n"
+            "- 'general_query': General insurance terminology or concepts\n\n"
+            "Return ONLY the category name."
+        )
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=query)])
+        intent = getattr(response, 'content', str(response)).lower().strip()
+        
+        valid_intents = ['list_plans', 'plan_details', 'compare_plans', 'recommendation', 'general_query']
+        if intent not in valid_intents:
+            intent = "plan_details"  # Default fallback
+            
+        return {"intent": intent}
+
+    # =========================================================================
+    # NODE 3: Entity Extractor
+    # =========================================================================
+    def entity_extractor_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Extracts structured entities from the query:
+        - provider (insurer names)
+        - insurance_type (term, ulip, savings, etc.)
+        - plan_names (specific plan names mentioned)
+        - user_profile (age, income, smoker, dependents, goal)
+        """
+        query = state["input"].lower()
+        
+        # Extract providers
+        provider_map = {
+            "edelweiss": "Edelweiss Life",
+            "tata": "TATA AIA",
+            "tata aia": "TATA AIA",
+            "generali": "Generali Central",
+            "central": "Generali Central",
+            "pramerica": "PRAMERICA"
+        }
+        providers = []
+        for keyword, name in provider_map.items():
+            if keyword in query and name not in providers:
+                providers.append(name)
+        
+        # Extract insurance types
+        type_map = {
+            "term": ["Term Insurance", "Term Plan"],
+            "ulip": ["Unit Linked Insurance Plan", "ULIP Plan"],
+            "wealth": ["Unit Linked Insurance Plan"],
+            "savings": ["Savings Plan", "Guaranteed Return"],
+            "retirement": ["Retirement and Pension"],
+            "pension": ["Retirement and Pension"],
+            "health": ["Health Insurance"],
+            "group": ["Group Plan"]
+        }
+        insurance_types = []
+        for keyword, types in type_map.items():
+            if keyword in query:
+                for t in types:
+                    if t not in insurance_types:
+                        insurance_types.append(t)
+        
+        # Extract specific plan names using LLM
+        plan_names = self._extract_plan_names_from_query(state["input"])
+        
+        # Extract user profile for recommendation intent
+        user_profile = {}
+        if state.get("intent") == "recommendation":
+            user_profile = self._extract_user_profile(state["input"])
+        
+        entities: ExtractedEntities = {
+            "provider": list(set(providers)) if providers else [],
+            "insurance_type": list(set(insurance_types)) if insurance_types else [],
+            "plan_names": list(set(plan_names)) if plan_names else [],
+            "user_profile": user_profile or {}
+        }
+        
+        # Build metadata filters from entities
+        filters = {}
+        if providers:
+            filters["insurer"] = providers
+        if insurance_types:
+            filters["insurance_type"] = insurance_types
+        
+        return {
+            "extracted_entities": entities,
+            "metadata_filters": filters
+        }
+
+    def _extract_plan_names_from_query(self, query: str) -> List[str]:
+        """Use LLM to extract specific plan names mentioned in query."""
+        llm = LLMFactory.get_llm("small")
+        
+        system_prompt = (
+            "Extract EXACT insurance plan names from the query.\n"
+            "If the user is asking to compare, extract BOTH plan names.\n"
+            "RULES:\n"
+            "- Return one plan name per line\n"
+            "- Include insurer prefix if mentioned (e.g., 'TATA AIA Smart Value Income', 'Edelweiss Saral Jeevan Bima')\n"
+            "- Return EMPTY if no specific plan names found\n"
+            "- Do NOT invent plan names"
+        )
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=query)])
+        result = getattr(response, 'content', str(response)).strip()
+        
+        # Validation: If LLM returns a sentence instead of names, skip it
+        if "mentioned in" in result.lower() or "referring to" in result.lower() or len(result) > 200:
+            return []
+        
+        if not result or result.lower() in ['none', 'empty', 'n/a']:
+            return []
+        
+        # Parse response
+        plan_names = []
+        for line in result.split('\n'):
+            line = re.sub(r'^[\d\.\-\*\u2022]\s*', '', line).strip().strip('"\'')
+            if len(line) > 5:
+                plan_names.append(line)
+        
+        return plan_names
+
+    def _extract_user_profile(self, query: str) -> Dict[str, Any]:
+        """Extract user profile information for recommendations."""
+        llm = LLMFactory.get_llm("small")
+        
+        system_prompt = (
+            "Extract user profile from the insurance query.\n"
+            "Return in format:\n"
+            "age: <number or null>\n"
+            "smoker: <yes/no or null>\n"
+            "cover_amount: <amount or null>\n"
+            "goal: <protection/savings/retirement/wealth or null>\n"
+            "dependents: <number or null>"
+        )
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=query)])
+        result = getattr(response, 'content', str(response))
+        
+        profile = {}
+        for line in result.split('\n'):
+            if ':' in line:
+                key, value = line.split(':', 1)
+                key = key.strip().lower()
+                value = value.strip().lower()
+                if value not in ['null', 'none', 'n/a', '']:
+                    if key == 'age':
+                        try:
+                            profile['age'] = int(re.search(r'\d+', value).group())
+                        except:
+                            pass
+                    elif key == 'smoker':
+                        profile['smoker'] = 'yes' in value
+                    elif key == 'cover_amount':
+                        profile['cover_amount'] = value
+                    elif key == 'goal':
+                        profile['goal'] = value
+                    elif key == 'dependents':
+                        try:
+                            profile['dependents'] = int(re.search(r'\d+', value).group())
+                        except:
+                            pass
+        
+        return profile
+
+    # =========================================================================
+    # NODE 4: Retrieval Router
+    # =========================================================================
+    def retrieval_router_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Determines retrieval strategy based on intent.
+        """
+        intent = state.get("intent", "plan_details")
+        
+        strategy_map = {
+            "list_plans": "metadata_only",
+            "plan_details": "plan_level",
+            "compare_plans": "cross_plan",
+            "recommendation": "section_specific",
+            "general_query": "plan_level"
+        }
+        
+        return {"retrieval_strategy": strategy_map.get(intent, "plan_level")}
+
+    # =========================================================================
+    # NODE 5: Retriever
+    # =========================================================================
+    def retriever_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Retrieves documents with:
+        - Metadata filtering
+        - CIS boosting for exclusions/charges/conditions
+        - Deduplication by similarity
+        """
+        retriever = self._get_retriever()
+        if not retriever:
+            return {"context": [], "retrieved_chunks": {}}
+        
+        query = state["input"]
+        filters = state.get("metadata_filters", {})
+        entities = state.get("extracted_entities", {})
+        strategy = state.get("retrieval_strategy", "plan_level")
+        
+        # If specific plan names were extracted, use them for precise retrieval
+        plan_names = entities.get("plan_names") or []
+        matched_plans = []
+        if plan_names:
+            # Resolve to actual plan names in index
+            all_plans = self._list_plans_from_index()
+            for name in plan_names:
+                match = self._find_closest_plan_name(name, all_plans)
+                if match:
+                    matched_plans.append(match)
+            
+            # Update filters for non-comparison queries (for comparison, _retrieve_for_comparison handles it)
+            if matched_plans and strategy != "cross_plan":
+                filters = filters.copy()
+                filters["product_name"] = matched_plans[0] if len(matched_plans) == 1 else matched_plans
+        
+        boost_cis = any(kw in query.lower() for kw in 
+                       ["exclusion", "excluded", "not covered", "charges", "fee", "condition", "waiting"])
+        
+        # Retrieve documents
+        if strategy == "cross_plan":
+            # For comparisons, retrieve for each plan separately
+            # Pass matched_plans if we have them, otherwise it will try to find them from filters
+            chunks_by_plan = self._retrieve_for_comparison(query, filters, entities, matched_plans=matched_plans)
+        else:
+            docs = retriever.search(query, filters=filters, k=8)
+            chunks_by_plan = self._group_by_plan_id(docs)
+        
+        # Boost CIS documents if needed
+        if boost_cis:
+            chunks_by_plan = self._boost_cis_chunks(chunks_by_plan)
+        
+        # Format context strings
+        limit_per_plan = 5 if strategy == "cross_plan" else 3
+        context = self._format_context(chunks_by_plan, limit=limit_per_plan)
+        
+        return {
+            "context": context,
+            "retrieved_chunks": chunks_by_plan
+        }
+
+    def _format_context(self, chunks_by_plan: Dict[str, List[Dict]], limit: int = 3) -> List[str]:
+        """Helper to format chunks into LLM-readable context strings."""
+        context = []
+        for plan_id, chunks in chunks_by_plan.items():
+            for chunk in chunks[:limit]:
+                content = chunk.get("content", "")
+                plan_name = chunk.get("product_name", "Unknown")
+                doc_type = chunk.get("document_type", "brochure")
+                section = chunk.get("section", "General")
+                context.append(f"[{plan_name} - {doc_type.upper()} - {section}] {content}")
+        return context
+
+    def _retrieve_for_comparison(self, query: str, filters: Dict, entities: Dict, matched_plans: List[str] = None) -> Dict[str, List]:
+        """Retrieve chunks for each plan separately in comparison mode."""
+        retriever = self._get_retriever()
+        if not retriever:
+            return {}
+        
+        if not matched_plans:
+            plan_names = entities.get("plan_names") or []
+            all_index_plans = self._list_plans_from_index()
+            matched_plans = []
+            for name in plan_names:
+                match = self._find_closest_plan_name(name, all_index_plans)
+                if match:
+                    matched_plans.append(match)
+        
+        if not matched_plans:
+            # Plan A: Deterministic "List & Match" Discovery
+            # For each provider, list all their plans and see if any match the query
+            providers = entities.get("provider") or []
+            
+            if not providers:
+                search_providers = [None]
+            else:
+                search_providers = providers
+                
+            discovered_names = []
+            all_plans_in_index = self._list_plans_from_index()
+            
+            for prov in search_providers:
+                prov_filter = {"insurer": prov} if prov else {}
+                prov_plans = self._list_plans_from_index(filters=prov_filter)
+                self._log_debug(f"Provider: {prov}, Plans found: {len(prov_plans)}")
+                
+                # Try to find which plan from this insurer is mentioned in the query
+                match = self._find_closest_plan_name(query, prov_plans)
+                self._log_debug(f"Match for {prov}: {match} (In list: {match in prov_plans})")
+                
+                if match and match in prov_plans and match not in discovered_names:
+                    discovered_names.append(match)
+            
+            matched_plans = discovered_names
+        
+        if not matched_plans:
+            # Plan B: Fall back to broad similarity-based discovery as a last resort
+            discovery_docs = retriever.search(query, k=20)
+            for d in discovery_docs:
+                p_name = d.metadata.get("product_name")
+                if p_name and p_name not in matched_plans:
+                    matched_plans.append(p_name)
+            matched_plans = matched_plans[:3]
+        
+        if not matched_plans:
+            # Plan B: Fall back to listing plans matching filters (metadata-only)
+            matched_plans = self._list_plans_from_index(filters)[:5]
+        
+        chunks_by_plan = defaultdict(list)
+        for matched in matched_plans:
+            
+            # Use a focused query for each plan instead of the broad comparison query
+            # This helps the retriever find relevant feature chunks for the specific plan
+            focused_query = f"features, benefits, eligibility and exclusions of {matched}"
+            
+            # Use a fresh, strictly focused filter for each plan
+            # IMPORTANT: Search by insurer and manually filter by product_name
+            # This is more robust than passing a combined filter to the vector store
+            matched_insurer = None
+            if hasattr(self, "_cached_plans") and self._cached_plans:
+                for p_meta in self._cached_plans:
+                    if p_meta["product_name"] == matched:
+                        matched_insurer = p_meta.get("insurer")
+                        break
+            
+            search_filters = {"insurer": matched_insurer} if matched_insurer else {}
+            
+            # Search only by insurer and then manually filter by product_name
+            # This is more robust than passing a combined filter to the vector store
+            docs = retriever.search(focused_query, filters=search_filters, k=50)
+            
+            plan_chunks = []
+            for doc in docs:
+                doc_product = doc.metadata.get("product_name", "")
+                # Use fuzzy match for manual filter consistency
+                if self._find_closest_plan_name(doc_product, [matched]) == matched:
+                    plan_chunks.append(doc)
+            
+            for doc in plan_chunks[:10]:
+                plan_id = doc.metadata.get("plan_id", matched)
+                chunks_by_plan[plan_id].append({
+                    "content": doc.page_content,
+                    "product_name": doc.metadata.get("product_name"),
+                    "document_type": doc.metadata.get("document_type", "brochure"),
+                    "section": doc.metadata.get("section", "General")
+                })
+        
+        return dict(chunks_by_plan)
+
+    def _group_by_plan_id(self, docs: List) -> Dict[str, List]:
+        """Group retrieved documents by plan_id."""
+        grouped = defaultdict(list)
+        for doc in docs:
+            plan_id = doc.metadata.get("plan_id", doc.metadata.get("product_name", "unknown"))
+            grouped[plan_id].append({
+                "content": doc.page_content,
+                "product_name": doc.metadata.get("product_name"),
+                "document_type": doc.metadata.get("document_type", "brochure"),
+                "section": doc.metadata.get("section", "General")
+            })
+        return dict(grouped)
+
+    def _boost_cis_chunks(self, chunks_by_plan: Dict[str, List]) -> Dict[str, List]:
+        """Boost CIS documents to appear first for each plan."""
+        boosted = {}
+        for plan_id, chunks in chunks_by_plan.items():
+            cis_chunks = [c for c in chunks if c.get("document_type") == "cis"]
+            brochure_chunks = [c for c in chunks if c.get("document_type") != "cis"]
+            boosted[plan_id] = cis_chunks + brochure_chunks
+        return boosted
+
+    # =========================================================================
+    # NODE 6: Plan Aggregator
+    # =========================================================================
+    def plan_aggregator_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Aggregates chunks by plan_id, merging brochure and CIS context.
+        CIS overrides brochure for exclusions, charges, conditions.
+        """
+        chunks_by_plan = state.get("retrieved_chunks", {})
+        
+        # Already grouped, just ensure proper ordering
+        aggregated = {}
+        for plan_id, chunks in chunks_by_plan.items():
+            # Separate by document type
+            cis_chunks = [c for c in chunks if c.get("document_type") == "cis"]
+            brochure_chunks = [c for c in chunks if c.get("document_type") != "cis"]
+            
+            # For exclusions/charges sections, prefer CIS
+            override_sections = ["Exclusions", "Charges", "Waiting Period", "Conditions"]
+            
+            final_chunks = []
+            covered_sections = set()
+            
+            # Add CIS chunks first for override sections
+            for chunk in cis_chunks:
+                section = chunk.get("section", "General")
+                if section in override_sections:
+                    final_chunks.append(chunk)
+                    covered_sections.add(section)
+            
+            # Add brochure chunks, skipping overridden sections
+            for chunk in brochure_chunks:
+                section = chunk.get("section", "General")
+                if section not in covered_sections:
+                    final_chunks.append(chunk)
+            
+            # Add remaining CIS chunks
+            for chunk in cis_chunks:
+                if chunk not in final_chunks:
+                    final_chunks.append(chunk)
+            
+            aggregated[plan_id] = final_chunks
+        
+        # Refresh context strings based on aggregated chunks
+        intent = state.get("intent", "plan_details")
+        limit = 5 if intent == "compare_plans" else 3
+        context = self._format_context(aggregated, limit=limit)
+        
+        return {
+            "retrieved_chunks": aggregated,
+            "context": context
+        }
+
+    # =========================================================================
+    # NODE 7: Listing Agent
+    # =========================================================================
+    def listing_agent(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Lists available plans based on filters.
+        Uses direct index access for accuracy.
+        """
+        llm = LLMFactory.get_llm("small")
+        query = state["input"]
+        filters = state.get("metadata_filters", {})
+        
+        plans = self._list_plans_from_index(filters)
+        plans = sorted(list(set(plans)))
+        
+        if not plans:
+            filter_desc = ", ".join([str(v) for v in filters.values()]) if filters else "your criteria"
+            answer = f"I couldn't find any plans matching {filter_desc}. Please try a different search."
+            return {"context": [], "answer": answer}
+        
+        plans_str = "\n".join([f"- {p}" for p in plans])
+        
+        # Describe the filters
+        filter_parts = []
+        if filters.get("insurer"):
+            insurer_list = filters["insurer"] if isinstance(filters["insurer"], list) else [filters["insurer"]]
+            filter_parts.append(f"from {', '.join(insurer_list)}")
+        if filters.get("insurance_type"):
+            type_list = filters["insurance_type"] if isinstance(filters["insurance_type"], list) else [filters["insurance_type"]]
+            filter_parts.append(f"in {', '.join(type_list)} category")
+        
+        filter_desc = " ".join(filter_parts) if filter_parts else ""
+        
+        system_prompt = (
+            "Present the following insurance plans in a clear, friendly manner.\n"
+            "RULES:\n"
+            "- ONLY include plans from the list below\n"
+            "- Group by insurer if multiple insurers present\n"
+            "- Use bullet points for clarity\n"
+            "- Do NOT mention technical details about data retrieval"
+        )
+        
+        prompt = f"User asked: {query}\n\nAvailable plans {filter_desc}:\n{plans_str}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        answer = getattr(response, 'content', str(response))
+        
+        return {"context": [f"Plans: {plans}"], "answer": answer}
+
+    # =========================================================================
+    # NODE 8: Plan Details Agent (Retrieval Agent)
+    # =========================================================================
+    def retrieval_agent(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Provides detailed information about a specific plan.
+        Grounds all responses in retrieved documents.
+        """
+        llm = LLMFactory.get_llm("medium")
+        query = state["input"]
+        context = state.get("context", [])
+        
+        if not context:
+            # Fallback retrieval
+            retriever = self._get_retriever()
+            if retriever:
+                docs = retriever.search(query, k=5)
+                context = [f"[{d.metadata.get('product_name')}] {d.page_content}" for d in docs]
+        
+        context_str = "\n\n".join(context)
+        
+        system_prompt = f"""You are an Insurance Policy Specialist providing accurate information.
+
+{COMPLIANCE_RULES}
+
+Answer the user's question using ONLY the Policy Context provided to you.
+If information is not in the context, say "I don't have that specific information in our documents."
+DO NOT mention that you are looking at documents or context. Just provide the answer.
+Be warm and helpful while maintaining accuracy."""
+        
+        prompt = f"Policy Context:\n{context_str}\n\nUser Question: {query}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        answer = getattr(response, 'content', str(response))
+        
+        return {"answer": answer}
+
+    # =========================================================================
+    # NODE 9: Comparison Agent
+    # =========================================================================
+    def comparison_agent(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Generates structured side-by-side comparisons.
+        Normalizes attributes across plans.
+        """
+        llm = LLMFactory.get_llm("medium")
+        query = state["input"]
+        context = state.get("context", [])
+        chunks_by_plan = state.get("retrieved_chunks", {})
+        
+        # Get plan names being compared
+        plan_names = list(chunks_by_plan.keys()) if chunks_by_plan else []
+        
+        if not context and not plan_names:
+            return {"answer": "I couldn't find the plans you want to compare. Please specify the plan names."}
+        
+        context_str = "\n\n".join(context)
+        plans_info = f"\n\nPlans to compare: {', '.join(plan_names)}" if plan_names else ""
+        
+        system_prompt = f"""You are an Insurance Comparison Expert.
+
+{COMPLIANCE_RULES}
+
+COMPARISON FORMAT:
+- Return comparison as a Markdown TABLE
+- Columns: Features | Plan 1 | Plan 2 | ...
+- Rows: Plan Type, Eligibility, Sum Assured, Premium Terms, Key Benefits, Exclusions
+- If a detail is missing, put "Not specified"
+- Include ALL plans mentioned in the context
+- Be objective and factual"""
+        
+        prompt = f"Policy Context:\n{context_str}{plans_info}\n\nUser Question: {query}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        answer = getattr(response, 'content', str(response))
+        
+        return {"answer": answer, "reasoning_output": f"Compared {len(plan_names)} plans"}
+
+    # =========================================================================
+    # NODE 10: Recommendation Agent (Advisory)
+    # =========================================================================
+    def advisory_agent(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Provides personalized recommendations based on user profile.
+        Grounds all advice in retrieved documents.
+        """
+        llm = LLMFactory.get_llm("large")
+        query = state["input"]
+        context = state.get("context", [])
+        entities = state.get("extracted_entities", {})
+        user_profile = entities.get("user_profile", {})
+        
+        context_str = "\n\n".join(context) if context else "No specific plans found matching your criteria."
+        
+        profile_info = ""
+        if user_profile:
+            profile_parts = []
+            if user_profile.get("age"):
+                profile_parts.append(f"Age: {user_profile['age']}")
+            if user_profile.get("smoker") is not None:
+                profile_parts.append(f"Smoker: {'Yes' if user_profile['smoker'] else 'No'}")
+            if user_profile.get("cover_amount"):
+                profile_parts.append(f"Cover needed: {user_profile['cover_amount']}")
+            if user_profile.get("goal"):
+                profile_parts.append(f"Goal: {user_profile['goal']}")
+            if profile_parts:
+                profile_info = f"\n\nUser Profile: {', '.join(profile_parts)}"
+        
+        system_prompt = f"""You are an Expert Insurance Advisor.
+
+{COMPLIANCE_RULES}
+
+RECOMMENDATION RULES:
+- Base recommendations ONLY on plans in the context
+- Consider user's age, smoking status, cover requirement if provided
+- Explain WHY a plan suits them based on document features
+- List 2-3 suitable options if available
+- Be clear about eligibility criteria
+- DO NOT reference the "context" or "documents" in your answer. Provide the advice directly."""
+        
+        prompt = f"Policy Context:\n{context_str}{profile_info}\n\nUser Question: {query}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        answer = getattr(response, 'content', str(response))
+        
+        return {"answer": answer}
+
+    # =========================================================================
+    # NODE 11: General Query Agent (FAQ)
+    # =========================================================================
+    def faq_agent(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Handles general insurance questions.
+        Still attempts to ground in documents when possible.
+        """
+        llm = LLMFactory.get_llm("small")
+        query = state["input"]
+        context = state.get("context", [])
+        
+        context_str = "\n\n".join(context) if context else ""
+        
+        system_prompt = f"""You are an Insurance Helpdesk Assistant.
+
+{COMPLIANCE_RULES}
+
+For general insurance terminology questions:
+- Provide accurate, helpful explanations
+- If context is available, use it to give specific examples
+- Keep explanations simple and jargon-free"""
+        
+        prompt = f"Context (if relevant):\n{context_str}\n\nUser Question: {query}" if context_str else f"User Question: {query}"
+        
+        response = llm.invoke([SystemMessage(content=system_prompt), HumanMessage(content=prompt)])
+        answer = getattr(response, 'content', str(response))
+        
+        return {"answer": answer}
+
+    # =========================================================================
+    # NODE 12: Guardrail
+    # =========================================================================
+    def guardrail_node(self, state: AgentState) -> Dict[str, Any]:
+        """
+        Final validation and compliance disclaimer.
+        - Validates answer is grounded
+        - Adds compliance disclaimer
+        - Blocks hallucinated content
+        """
+        answer = state.get("answer", "")
+        
+        if not answer:
+            answer = "I apologize, but I couldn't generate a response. Please try rephrasing your question."
+        
+        # Add compliance disclaimer
+        if COMPLIANCE_DISCLAIMER not in answer:
+            answer = answer + COMPLIANCE_DISCLAIMER
+        
+        return {"answer": answer}
+
+    # =========================================================================
+    # HELPER METHODS
+    # =========================================================================
+    def _list_plans_from_index(self, filters: Dict = None) -> List[str]:
+        """Returns unique product names matching filters. Optimized with caching."""
+        retriever = self._get_retriever()
+        if not retriever:
+            return []
+        
+        try:
+            # Use a simple cache attribute on the instance if it doesn't exist
+            if not hasattr(self, "_cached_plans") or self._cached_plans is None:
+                store = retriever.vector_store
+                plans_metadata = []
+                for doc in store.docstore._dict.values():
+                    p_name = doc.metadata.get('product_name')
+                    insurer = doc.metadata.get('insurer')
+                    i_type = doc.metadata.get('insurance_type')
+                    if p_name:
+                        plans_metadata.append({
+                            "product_name": p_name,
+                            "insurer": insurer,
+                            "insurance_type": i_type
+                        })
+                self._cached_plans = plans_metadata
+
+            # Filter from cache
+            plans = set()
+            for meta in self._cached_plans:
+                if filters:
+                    match = True
+                    for k, v in filters.items():
+                        doc_val = str(meta.get(k, "")).lower().strip()
+                        if not doc_val:
+                            match = False
+                            break
+                            
+                        # Standardize filter values to list of lowercase strings
+                        filter_values = v if isinstance(v, list) else [v]
+                        filter_values = [str(fv).lower().strip() for fv in filter_values]
+                        
+                        # Robust match: any filter item matches or is matched by doc_val
+                        val_match = False
+                        for fv in filter_values:
+                            if k == "product_name":
+                                if fv in doc_val or doc_val in fv:
+                                    val_match = True
+                                    break
+                            elif k == "insurer": # Strictly match insurer names
+                                if fv == doc_val:
+                                    val_match = True
+                                    break
+                            else: # For other keys like insurance_type, allow exact match
+                                if fv == doc_val:
+                                    val_match = True
+                                    break
+                        
+                        if not val_match:
+                            match = False
+                            break
+                            
+                    if not match:
+                        continue
+                plans.add(meta["product_name"])
+            
+            return sorted(list(plans))
+        except Exception:
+            return []
+
+    def _find_closest_plan_name(self, query_plan: str, all_plans: List[str]) -> Optional[str]:
+        """Finds closest matching plan name using fuzzy matching."""
+        if not all_plans:
+            return query_plan
+            
+        def normalize(s):
+            return s.lower().replace(" ", "").replace("-", "").replace("_", "").replace("edelweisslife", "edelweiss")
+            
+        query_norm = normalize(query_plan)
+        
+        # 1. Exact match (case insensitive)
+        for plan in all_plans:
+            if plan.lower() == query_plan.lower():
+                return plan
+                
+        # 2. Normalized containment match (High Confidence)
+        # Check if the plan name is mentioned in the query
+        for plan in all_plans:
+            plan_norm = normalize(plan)
+            if plan_norm in query_norm or query_norm in plan_norm:
+                return plan
+        
+        # 3. Word overlap (Lower Confidence fallback)
+        query_words = set(query_plan.lower().split())
+        stop_words = {"tata", "aia", "edelweiss", "life", "generali", "central", "plan", "insurance", "the", "a", "of", "with", "compare"}
+        query_significant = query_words - stop_words
+        
+        best_match = None
+        max_overlap = 0
+        
+        for plan in all_plans:
+            plan_words = set(plan.lower().split())
+            plan_significant = plan_words - stop_words
+            
+            # Count significant word overlap
+            overlap = len(query_significant.intersection(plan_significant))
+            
+            if overlap > max_overlap:
+                max_overlap = overlap
+                best_match = plan
+                
+        # Return best match if we found significant overlap (at least 2 words)
+        return best_match if max_overlap >= 2 else query_plan
+
+
+# Singleton instance
+nodes = AgentNodes()

agents/states.py

@@ -0,0 +1,51 @@
+from typing import TypedDict, Annotated, List, Dict, Any, Optional
+import operator
+
+
+class UserProfile(TypedDict, total=False):
+    """User profile for recommendation intent."""
+    age: Optional[int]
+    income: Optional[str]
+    smoker: Optional[bool]
+    dependents: Optional[int]
+    goal: Optional[str]  # "protection", "savings", "retirement", "wealth"
+    cover_amount: Optional[str]  # e.g., "1 Cr", "50 Lakh"
+
+
+class ExtractedEntities(TypedDict, total=False):
+    """Entities extracted from user query."""
+    provider: Optional[List[str]]  # ["TATA AIA", "Edelweiss Life"]
+    insurance_type: Optional[List[str]]  # ["Term Insurance", "ULIP"]
+    plan_names: Optional[List[str]]  # Specific plan names mentioned
+    user_profile: Optional[UserProfile]
+
+
+class AgentState(TypedDict):
+    """
+    Enhanced state for the LangGraph RAG workflow.
+    Supports deterministic, compliance-focused retrieval.
+    """
+    # Input
+    input: str
+    chat_history: List[str]
+    
+    # Query Classification
+    intent: str  # 'list_plans', 'plan_details', 'compare_plans', 'recommendation', 'general_query'
+    
+    # Entity Extraction
+    extracted_entities: ExtractedEntities
+    
+    # Retrieval Configuration
+    metadata_filters: Dict[str, Any]  # Filters for vector store
+    retrieval_strategy: str  # 'metadata_only', 'plan_level', 'section_specific', 'cross_plan'
+    
+    # Retrieved Content
+    context: List[str]  # accumulated context strings
+    retrieved_chunks: Dict[str, List[Dict]]  # Grouped by plan_id: {plan_id: [chunks]}
+    
+    # Reasoning & Output
+    reasoning_output: str  # Structured comparison/recommendation data
+    answer: str  # Final answer to user
+    
+    # Internal Routing
+    next_step: str  # For conditional edges

api/main.py

@@ -0,0 +1,52 @@
+from fastapi import FastAPI, HTTPException, Body
+from pydantic import BaseModel
+from typing import List, Dict, Optional, Any
+from agents.graph import app as agent_app
+# from ingestion.pipeline import IngestionPipeline # Optional: Trigger via API
+
+app = FastAPI(title="Insurance Advisory AI Agent", version="1.0.0")
+
+class ChatRequest(BaseModel):
+    message: str
+    chat_history: Optional[List[str]] = []
+
+class ChatResponse(BaseModel):
+    answer: str
+    intent: str
+    context_used: Optional[List[str]] = None
+
+@app.get("/")
+def health_check():
+    return {"status": "active", "system": "Insurance Advisory Agent"}
+
+@app.post("/chat", response_model=ChatResponse)
+async def chat_endpoint(request: ChatRequest):
+    """
+    Main chat endpoint. Routes query through the Multi-Agent Graph.
+    """
+    try:
+        # Initial state
+        initial_state = {
+            "input": request.message,
+            "chat_history": request.chat_history or [],
+            "intent": "",
+            "context": [],
+            "answer": "",
+            "metadata_filters": {}
+        }
+        
+        # Invoke Graph
+        result = agent_app.invoke(initial_state)
+        
+        return ChatResponse(
+            answer=result.get("answer", "No response generated."),
+            intent=result.get("intent", "unknown"),
+            context_used=result.get("context", [])
+        )
+        
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app.py

@@ -0,0 +1,266 @@
+import os
+import json
+import threading
+import speech_recognition as sr
+from flask import Flask, render_template, request, jsonify
+from agents.graph import app as agent_app
+from dotenv import load_dotenv
+from ingestion.pipeline import IngestionPipeline
+from rag.vector_store import VectorStoreManager
+
+load_dotenv()
+
+app = Flask(__name__)
+
+# Global state for ingestion tracking
+ingestion_status = {
+    "status": "Idle",
+    "progress": 0,
+    "last_error": None
+}
+
+def ingest_worker(file_path, delete_source=None):
+    """Worker thread for background ingestion using enhanced pipeline."""
+    global ingestion_status
+    try:
+        ingestion_status["status"] = "Starting..."
+        ingestion_status["progress"] = 0
+        
+        base_docs_dir = "docs"
+        pipeline = IngestionPipeline(base_docs_dir)
+        vector_manager = VectorStoreManager()
+        
+        if delete_source:
+            ingestion_status["status"] = "Removing old version..."
+            vector_manager.delete_documents_by_source(delete_source)
+            ingestion_status["progress"] = 10
+        
+        ingestion_status["status"] = "Processing document..."
+        ingestion_status["progress"] = 30
+        
+        # Use the new unified process_single_file method
+        # Handles metadata extraction, section detection, and proper chunking
+        chunks = pipeline.process_single_file(file_path)
+        
+        if chunks:
+            ingestion_status["status"] = "Updating Vector Store..."
+            ingestion_status["progress"] = 70
+            vector_manager.update_vector_store(chunks)
+            
+            # Reload the retriever in the agent nodes to see new documents
+            from agents.nodes import nodes
+            nodes.reload_retriever()
+            
+            ingestion_status["status"] = "Completed Successfully!"
+            ingestion_status["progress"] = 100
+        else:
+            ingestion_status["status"] = "Failed: No content extracted."
+            ingestion_status["progress"] = 0
+            
+    except Exception as e:
+        ingestion_status["status"] = "Failed"
+        ingestion_status["last_error"] = str(e)
+        ingestion_status["progress"] = 0
+
+@app.route("/")
+def index():
+    return render_template("index.html")
+
+@app.route("/api/chat", methods=["POST"])
+def chat():
+    data = request.json
+    prompt = data.get("prompt")
+    history = data.get("history", [])
+    
+    if not prompt:
+        return jsonify({"error": "Prompt is required"}), 400
+        
+    try:
+        initial_state = {
+            "input": prompt,
+            "chat_history": history,
+            "intent": "",
+            "context": [],
+            "answer": "",
+            "metadata_filters": {}
+        }
+        
+        result = agent_app.invoke(initial_state)
+        return jsonify({
+            "answer": result.get("answer", ""),
+            "context": result.get("context", [])
+        })
+    except Exception as e:
+        return jsonify({"error": str(e), "status": "error"}), 500
+
+@app.route("/api/audio-chat", methods=["POST"])
+def audio_chat():
+    if 'audio' not in request.files:
+        return jsonify({"error": "No audio file part"}), 400
+    
+    file = request.files['audio']
+    history = json.loads(request.form.get("history", "[]"))
+    
+    if file.filename == '':
+        return jsonify({"error": "No selected file"}), 400
+
+    temp_path = "temp_voice_query.wav"
+    file.save(temp_path)
+    
+    r = sr.Recognizer()
+    try:
+        with sr.AudioFile(temp_path) as source:
+            audio_data = r.record(source)
+        
+        raw_text = r.recognize_google(audio_data)
+        
+        # Summarize/Refine the transcribed audio text
+        from models.llm import LLMFactory
+        from langchain_core.messages import SystemMessage, HumanMessage
+        
+        refiner_llm = LLMFactory.get_llm("small")
+        refine_system = (
+            "You are an assistant that cleans up and summarizes noisy speech-to-text transcriptions. "
+            "Your goal is to extract the actual insurance-related question or request from the text.\n\n"
+            "RULES:\n"
+            "1. Remove filler words (um, ah, like, you know).\n"
+            "2. Fix grammatical errors caused by transcription.\n"
+            "3. If multiple things are mentioned, focus on the core request.\n"
+            "4. Return ONLY the cleaned, professional question text."
+        )
+        
+        refine_response = refiner_llm.invoke([
+            SystemMessage(content=refine_system),
+            HumanMessage(content=f"Transcription: {raw_text}")
+        ])
+        summarized_text = getattr(refine_response, 'content', str(refine_response)).strip()
+        
+        # Process with existing Agent using the summarized text
+        initial_state = {
+            "input": summarized_text,
+            "chat_history": history,
+            "intent": "",
+            "context": [],
+            "answer": "",
+            "metadata_filters": {}
+        }
+        
+        result = agent_app.invoke(initial_state)
+        
+        if os.path.exists(temp_path):
+            os.remove(temp_path)
+            
+        return jsonify({
+            "transcription": raw_text,
+            "summarized_question": summarized_text,
+            "answer": result.get("answer", ""),
+            "context": result.get("context", [])
+        })
+        
+    except sr.UnknownValueError:
+        if os.path.exists(temp_path): os.remove(temp_path)
+        return jsonify({"error": "Could not understand audio"}), 400
+    except sr.RequestError as e:
+        if os.path.exists(temp_path): os.remove(temp_path)
+        return jsonify({"error": f"Speech service error: {e}"}), 500
+    except Exception as e:
+        if os.path.exists(temp_path): os.remove(temp_path)
+        return jsonify({"error": str(e)}), 500
+
+def update_doc_structure(provider_name, category_name):
+    """Helper to persist new providers/categories to the config file."""
+    try:
+        config_path = os.path.join("configs", "doc_structure.json")
+        if not os.path.exists(config_path):
+            return
+        
+        with open(config_path, "r") as f:
+            config = json.load(f)
+        
+        # Find or create provider
+        provider = next((p for p in config["providers"] if p["name"] == provider_name), None)
+        if not provider:
+            # Insert at the beginning (before 'Other')
+            provider = {"name": provider_name, "categories": []}
+            config["providers"].insert(0, provider)
+        
+        # Add category if new
+        if category_name not in provider["categories"]:
+            provider["categories"].append(category_name)
+            # Sort categories for cleanliness (except if it was General)
+            if len(provider["categories"]) > 1:
+                provider["categories"].sort()
+        
+        with open(config_path, "w") as f:
+            json.dump(config, f, indent=4)
+    except Exception as e:
+        pass
+
+@app.route("/api/upload", methods=["POST"])
+def upload():
+    if 'file' not in request.files:
+        return jsonify({"error": "No file part"}), 400
+    
+    file = request.files['file']
+    provider = request.form.get("provider")
+    category = request.form.get("category")
+    mode = request.form.get("mode", "New Upload") # "New Upload" or "Modify Existing"
+    
+    if file.filename == '' or not provider or not category:
+        return jsonify({"error": "Missing metadata or file"}), 400
+
+    # Persist new structure to JSON
+    update_doc_structure(provider, category)
+
+    base_dir = "docs"
+    target_dir = os.path.join(base_dir, provider, category)
+    os.makedirs(target_dir, exist_ok=True)
+    
+    file_path = os.path.join(target_dir, file.filename)
+    file.save(file_path)
+    
+    delete_source = None
+    if mode == "Modify Existing":
+        file_to_modify = request.form.get("file_to_modify")
+        if file_to_modify:
+            delete_source = os.path.join(base_dir, provider, category, file_to_modify)
+            if os.path.abspath(delete_source) != os.path.abspath(file_path):
+                if os.path.exists(delete_source):
+                    os.remove(delete_source)
+
+    # Start background ingestion
+    thread = threading.Thread(target=ingest_worker, args=(file_path, delete_source))
+    thread.start()
+    
+    return jsonify({"message": "File uploaded, ingestion started.", "path": file_path})
+
+@app.route("/api/status", methods=["GET"])
+def get_status():
+    return jsonify(ingestion_status)
+
+@app.route("/api/config", methods=["GET"])
+def get_config():
+    config_path = os.path.join("configs", "doc_structure.json")
+    if os.path.exists(config_path):
+        with open(config_path, "r") as f:
+            return jsonify(json.load(f))
+    return jsonify({"providers": []})
+
+@app.route("/api/files", methods=["GET"])
+def list_files():
+    provider = request.args.get("provider")
+    category = request.args.get("category")
+    
+    if not provider or not category:
+        return jsonify({"files": []})
+        
+    base_dir = "docs"
+    target_dir = os.path.join(base_dir, provider, category)
+    if os.path.exists(target_dir):
+        files = [f for f in os.listdir(target_dir) if f.lower().endswith(('.pdf', '.docx'))]
+        return jsonify({"files": files})
+    return jsonify({"files": []})
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", 8000))
+    app.run(host="0.0.0.0", port=port)

configs/doc_structure.json

@@ -0,0 +1,82 @@
+{
+    "providers": [
+        {
+            "name": "Canara HSBC",
+            "categories": [
+                "Term Plan",
+                "Retirement Plan",
+                "ULIP Plan"
+            ]
+        },
+        {
+            "name": "Aviva",
+            "categories": [
+                "Term Plan",
+                "Savings Plan",
+                "ULIP Plan"
+            ]
+        },
+         {
+            "name": "Bharti Axa",
+            "categories": [
+                "Ulip Plan",
+                "Retirement Plan",
+                "Saving Plan"
+            ]
+        },
+        {
+            "name": "PRAMERICA",
+            "categories": [
+                "ULIP Plan",
+                "Health Plan",
+                "Retirement Plan",
+                "Savings Plan",
+                "Term Plan"
+            ]
+        },
+        {
+            "name": "TATA AIA",
+            "categories": [
+                "Combo",
+                "Health Plan",
+                "Retirement and Pension Plan",
+                "Savings Plan",
+                "ULIP Plans",
+                "Term Plans"
+            ]
+        },
+         {
+            "name": "IndusInd Nippon",
+            "categories": [
+                "Combo Plan",
+                "Pension Plan",
+                "Savings Plan",
+                "ULIP Plan",
+                "Term Plan"
+            ]
+        },
+        {
+            "name": "Edelweiss Life",
+            "categories": [
+                "Term Insurance Plans",
+                "Group Solutions",
+                "Guaranteed Income Plans",
+                "Micro Plans",
+                "Wealth Creation Plans"
+            ]
+        },
+        {
+            "name": "Generali Central",
+            "categories": [
+                "Term Plan",
+                "Savings Plan",
+                "ULIP Plan",
+                "Pension Plan"
+            ]
+        },
+        {
+            "name": "Other",
+            "categories": []
+        }
+    ]
+}

configs/plan_metadata.json

@@ -0,0 +1,103 @@
+{
+    "EdelweissLife_Wealth_Rise_Plus": {
+        "category": [
+            "Savings Plan",
+            "Unit Linked Insurance Plan"
+        ]
+    },
+    "EdelweissLife_Saral_Jeevan_Bima": {
+        "category": [
+            "Term Insurance"
+        ]
+    },
+    "EdelweissLife_Zindagi_Protect_Plus": {
+        "category": [
+            "Term Insurance"
+        ]
+    },
+    "EdelweissLife_Group_Employee_Benefit_Plus": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Pradhan_Mantri_Jeevan_Jyoti_Bima_Yojana": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Group_Life_Protection": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Group_Total_Secure": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Flexi_Dream_Plan": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Bharat_Savings_STAR": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Flexi_Savings_Plan": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Guaranteed_Flexi_STAR": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Guaranteed_Income_STAR": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Guaranteed_Savings_STAR": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Premier_Guaranteed_STAR": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Premier_Guaranteed_STAR_PRO": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Jan_Suraksha": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Raksha_Kavach_Bima_Yojana": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Wealth_Plus": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Wealth_Premier": {
+        "category": [
+            "Uncategorized"
+        ]
+    },
+    "EdelweissLife_Insurance_Wealth_Ultima": {
+        "category": [
+            "Uncategorized"
+        ]
+    }
+}

docs/Aviva/Savings Plan/Aviva Nivesh Bima Brochure.PDF

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9da29e17e6d1f071e9b5eaed05d0f58650de6fce84316c55c630b9faf2eb31d4
+size 886262

docs/Aviva/Savings Plan/Aviva Nivesh Bima CIS.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97d68bd2d7f6060cd9a2eccbeb47c98c2dab1e4de24813e5ccd52d0da0d82073
+size 25227

docs/Aviva/Savings Plan/Aviva PoS Dhan Suraksha Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e69f836f684f144bb5b8c3f624b2fe0774a1e3437d6c4234d4f6253dde79c39
+size 801522

docs/Aviva/Savings Plan/Aviva PoS Dhan Suraksha CIS.docx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15d7238105b4218340066971f8c64e719ebf486a38d2db005822795d5de1b892
+size 29354

docs/Aviva/Term Plan/Aviva Protection Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58e317ee65d45347aff75de325df45b9758e82e1cc20f84f1944d03f1a0624bd
+size 287843

docs/Aviva/ULIP Plan/Aviva Fortune Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1da0950c99b7d786ebc2d1359f1aa52334164b93eb955cfab68001a17b802342
+size 650527

docs/Bharti Axa/Retirement Plan/Bharti Axa Swabhimaan Retirement Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c1a138c1205a2807b7df04342f08a8a6b986ed2d75b807974b421fe20f64767
+size 1687191

docs/Bharti Axa/Saving Plan/Bharti Axa Guaranteed Bachat Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c39fb2a0ff4ea5d2ad70ef1484dc2483a68e064afa9b618efe7904a0298c2933
+size 3071848

docs/Bharti Axa/Saving Plan/Bharti Axa Guaranteed Wealth Pro Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7d1037a5f6d4bcb59d6f432ddabaf295104f1b81528cb51f8344a49c5030595
+size 679317

docs/Bharti Axa/Saving Plan/Bharti Axa Secure Insta Income Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62d53318a13b8507913c3da4d146d8c48cc8426d9b0e16782aa73ee20c56d614
+size 2013163

docs/Bharti Axa/Saving Plan/Bharti Axa Unnati Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02583b7c27c7514f2616f92f6e90c710a7e6a405cd7695e641fb3ffe73711a07
+size 644320

docs/Bharti Axa/Ulip Plan/Bhart Axa Growth Shield Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f25d8efd733c20956e2f6fb5cc124d48fcbba315fcdb587670fb46318a05a936
+size 668695

docs/Bharti Axa/Ulip Plan/Bharti Axa Life Dream Shield Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e654c94b96e3be9db9916ffc7cf21557206680aeb92a7b46171386f685603d9
+size 1155895

docs/Bharti Axa/Ulip Plan/Bharti Axa Unnati Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02583b7c27c7514f2616f92f6e90c710a7e6a405cd7695e641fb3ffe73711a07
+size 644320

docs/Canara HSBC/Retirement Plan/Canara HSBC EZ Pension Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:394a3f50d6c8cb17c73294e4640071c2e4dafd2c37036fbdf1e7105a882bd54c
+size 2471470

docs/Canara HSBC/Retirement Plan/Canara HSBC Pension 4 Life Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3bc2d20809919e8fc62e23f74a84c35474c4a20eee5c23631eea2e24e691336
+size 597661

docs/Canara HSBC/Retirement Plan/Canara HSBC iSelect Guaranteed Future Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61c808b23e6ea9e25e0c99b37034374f533de439dd248ba2446ea7e00289d161
+size 1515545

docs/Canara HSBC/Term Plan/Canara HSBC Promise 2 Protect Term Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06532d990882d27b61e2835f5d8412808fb0a3927558712dbba0fb9bf71dcb32
+size 839879

docs/Canara HSBC/Term Plan/Canara HSBC Young Term Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3043dff7ba993fac87dd48c705afa1ae28787e58fe9b3d255936a87a4cadbf72
+size 821708

docs/Canara HSBC/Term Plan/Canara HSBC iSelect Smart 360 Term Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc12bb1301a51dbdc963c55d58d776ed771b395c9a9226c963af3cd68c02277b
+size 871970

docs/Canara HSBC/ULIP Plan/Canara HSBC Promise 4 Growth Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7a8951e15e50a2deb9e2d84a8b3ea1d385a3fb27e60c113666c1af1a1bd8c52
+size 1068461

docs/Canara HSBC/ULIP Plan/Canara HSBC Secure Invest Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970f0240da50c6511aa8d9c33a551ac58d018954231877d5774685ecebb02aaf
+size 915741

docs/Canara HSBC/ULIP Plan/Canara HSBC Wealth Edge Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a66c2727b907e24cd520e47ce5d9a4854c5d0c9f026a919cd995c76a824be1
+size 4263484

docs/Edelweiss Life/Group Solutions/EdelweissLife Group Employee Benefit Plus Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb77163dfc4623112bece207f55d320d1b1349b63719723f89bbfb9b2edea900
+size 358074

docs/Edelweiss Life/Group Solutions/EdelweissLife Group Life Protection Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b58cfa8e526cab19e73f530a70333286ba58a71f60563fbf142a97704eece55
+size 289919

docs/Edelweiss Life/Group Solutions/EdelweissLife Group Life Protection CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:961d85be4cc51bcf42f37b9a6b0fe251b0950fb397a0bd3d91a4cd62ce1a7007
+size 193804

docs/Edelweiss Life/Group Solutions/EdelweissLife Group Total Secure Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bde2b7b8a2dacfef43b89ec30ebb0ff015d90e46a692096518d60d0fd905782
+size 420074

docs/Edelweiss Life/Group Solutions/EdelweissLife Pradhan Mantri Jeevan Jyoti Bima Yojana Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cca1758cd0d4738fdb8e1bf4d026f0107842abf6211ab335dcdb55741faa08f4
+size 334737

docs/Edelweiss Life/Group Solutions/EdelweissLife Pradhan Mantri Jeevan Jyoti Bima Yojana CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f048aed0c55c69399fb9713182e1aec0e39d066e5a0e500cd5278bc97ff68273
+size 177614

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Bharat Savings STAR Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:731ad161b10fa18967176a972ef4c899dac7a4743c4fbe52c94a997ea13a4006
+size 493775

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Bharat Savings STAR CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c409818e932309a79f3489c7e36c4533a76ccde1b4df832f705ebce5752f19b0
+size 230194

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Flexi Dream Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:729d4eec3800236ff14ac7e367d4f9b0a3c1fd27da22c3844a2d319c07a21cb8
+size 579303

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Flexi Dream Plan CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f32e3ee795d218481e04b12cee2baaf3207095dd734bc2e33ede2867a66629
+size 345849

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Flexi Savings Plan Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0740ae4ce24965183c66b3c6cd8b5325187cd38739c649710c5af6e21ea8f1c3
+size 513027

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Flexi Savings Plan CIS .pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d2d629ea0160f5619a74e344faf51bd03cb1d2ce22fd1f25d1faee6a6984c3a
+size 263599

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Guaranteed Flexi STAR Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae3d1dc3ccdb5f56044f0f81f492827847c04c53d8289652e2ea23b0ede9c6c2
+size 420020

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Guaranteed Flexi STAR CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76cd977f1887b63dd413e7797961ad01b5d3c4714925bbcfc1f65a65221fca7b
+size 268535

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Guaranteed Income STAR Brochure.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:896453b156d6823e4beb68197ba51bdada11b240afeca3587279637e93805b03
+size 615785

docs/Edelweiss Life/Guaranteed Income Plans/EdelweissLife Guaranteed Income STAR CIS.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbbe5519a6bb6f79638eebbe3bf137b4729bf3eda4d0c1a35fdc8ff006c549b1
+size 248372