Upload 37 files

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/raw_docs/train_station/EXP-TRAINS.json filter=lfs diff=lfs merge=lfs -text
+data/raw_docs/train_station/PASS-TRAINS.json filter=lfs diff=lfs merge=lfs -text
+data/vector_store/rules_faiss.index filter=lfs diff=lfs merge=lfs -text
+data/vector_store/rules_metadata.json filter=lfs diff=lfs merge=lfs -text

Chatbot/bot.py

@@ -0,0 +1,167 @@
+# Main program...Combines all modules
+
+import re
+from Chatbot.router import route_query
+from helpers.live_sources import retrieve_live_sources
+from dotenv import load_dotenv
+load_dotenv()
+HIGH_CONF = 0.75
+LOW_CONF = 0.45
+LINK_RELEVANCE_MIN = 0.01
+RELEVANCE_CLOSE_DELTA = 0.10
+
+# Calling different modules
+def call_railway_rag(query):
+    from modules.railway_rag.railway_base_rag import answer_with_rag
+    return answer_with_rag(query)
+def call_live_data(query):
+    from modules.live_data_apis import answer_with_live_data
+    return answer_with_live_data(query)
+def call_general(query,route,mode="module"):
+    from modules.general_chat import answer_general_query
+    return answer_general_query(query,route,mode)
+def call_link_answer(query, num_of_links):
+    from modules.link_answer import run
+    src=run(query,num_of_links)
+    if not src:
+        return {"answer": None, "has_answer": False, "meta": {}}
+    return {
+        "answer": [s["url"] for s in src],
+        "has_answer": True,
+        "meta": {
+            "status": "ok",
+            "type": "links_only",
+            "requested": num_of_links,
+            "returned": len(src)
+        }
+    }
+
+# Preliminary query check
+def analyze_input_structure(query: str):
+    q = query.strip()
+    if not q:
+        return "empty"
+    if re.fullmatch(r"[^\w\s]+", q):
+        return "noise"
+    return "normal"
+#Gets num of links
+def extract_num_links(query, default=2, max_links=10):
+    match = re.search(r"\b(\d+)\b", query)
+    if match:
+        return min(int(match.group(1)), max_links)
+    return default
+# Link adding reasons
+def link_reason(reason_key: str):
+    return {
+        "rag_moderate": ("This answer is based on available railway rules. For official confirmation or additional details, please refer to:"),
+        "api_stale": ("The available data may be outdated. For the latest official information, please refer to:"),
+        "api_unknown": ("For authoritative and up-to-date information, please refer to:"),
+        "general_info": ("For more detailed or official information, you may also refer to:"),
+        "API_not_working":("The API is currently not working. For more information, please refer to:")
+    }.get(reason_key, "For more information, please refer to:")
+
+# Clean links
+def format_live_sources(sources, reason):
+    text = "\n\n" + reason + "\n"
+    for s in sources:
+        text += s["url"] + "\n"
+    return text
+
+#Main
+def answer_query(query):
+    structure = analyze_input_structure(query)
+    if structure == "empty":
+        return "Please enter a query so I can help you."
+    if structure == "noise":
+        return "I couldn’t understand the input. Please enter a clear question."
+    # Calls router
+    route = route_query(query)
+    if route.get("router_failed"):
+        return call_general(query,route,mode="failsafe")["answer"]
+    preferences = route["module_preferences"]
+    relevance_map = {m["module"]: m["relevance"] for m in preferences}
+    tried = set()
+    for pref in preferences:
+        module = pref["module"]
+        if module in tried:
+            continue
+        tried.add(module)
+        # Link answer(links only)
+        if module == "link_answer":
+            num_links = extract_num_links(query)
+            result = call_link_answer(query, num_links)
+            if result.get("has_answer"):
+                return "\n".join(result["answer"])
+            continue
+        # RAG
+        if module == "railway_rag":
+            result=call_railway_rag(query)
+            if result.get("has_answer")==False:
+                continue
+            conf=result.get("meta",{}).get("confidence",0.0)
+            if conf >= HIGH_CONF:
+                return result["answer"]
+            if LOW_CONF <= conf < HIGH_CONF:
+                if relevance_map.get("link_answer",0) > LINK_RELEVANCE_MIN:
+                    links = retrieve_live_sources(query)
+                    if links:
+                        return (
+                            result["answer"]
+                            + format_live_sources(
+                                links,
+                                reason=link_reason("rag_moderate")
+                            )
+                        )
+                return result["answer"]
+            continue  
+        # Live data(APIs)
+        if module == "live_data_apis":
+            result=call_live_data(query)
+            if result.get("has_answer")==False and result.get("meta",{}).get("status")=="nothing":
+                continue
+            if result.get("has_answer")==False and result.get("meta",{}).get("status")=="api_failed":
+                links=retrieve_live_sources(query)
+                reason="API_not_working"
+                if links:
+                    return (format_live_sources(links,reason=link_reason(reason)))
+            if result.get("has_answer")==False and result.get("meta",{}).get("status")=="need_input":
+                FIELD_LABELS = {
+                "train_number": "Train number",
+                "from_station": "Source station",
+                "to_station": "Destination station",
+                "date": "Journey date",
+                "class_type": "Class (e.g., SL, 3A, 2A)",
+                "quota": "Quota (GN / Tatkal)",
+                "station_code": "Station name",
+                "pnr": "PNR number",
+                "hours": "Time window (in hours)"}
+                missing=result.get("meta",{}).get("missing_fields",[])
+                readable = [
+                    FIELD_LABELS.get(f, f.replace("_", " ").title())
+                    for f in missing]
+                return ("I need a bit more information to answer this.\n""Missing details: " + ", ".join(readable))
+            if result.get("has_answer")==True and result.get("meta",{}).get("status")=="ok":    
+                freshness = result.get("meta", {}).get("freshness", "unknown")
+                fallback_used = result.get("meta", {}).get("fallback_used", False)
+                if freshness == "fresh" and not fallback_used:
+                    return result["answer"]
+                if relevance_map.get("link_answer", 0) > LINK_RELEVANCE_MIN:
+                    reason=("api_stale" if freshness == "stale" else "api_unknown")
+                    links=retrieve_live_sources(query)
+                    if links:
+                        return (result["answer"]+ format_live_sources(links,reason=link_reason(reason)))
+            continue
+        # General
+        if module=="general":
+            result=call_general(query,route,mode="module")
+            gen_rel = relevance_map.get("general", 0)
+            src_rel = relevance_map.get("live_sources", 0)
+            if (src_rel > LINK_RELEVANCE_MIN and abs(gen_rel - src_rel) <= RELEVANCE_CLOSE_DELTA):
+                links = retrieve_live_sources(query)
+                if links:
+                    return (result["answer"]+ format_live_sources(links,reason=link_reason("general_info")))
+            if result.get("has_answer"):
+                return result["answer"]
+            continue
+    # Last failsafe
+    return call_general(query,route,mode="failsafe")["answer"]

Chatbot/router.py

@@ -0,0 +1,92 @@
+import os
+import json
+from openai import OpenAI
+from dotenv import load_dotenv
+load_dotenv()
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+if not client:
+    raise RuntimeError("OPENAI_API_KEY is not set")
+MODEL_NAME = "gpt-4.1-mini"
+
+ROUTER_SYSTEM_PROMPT = (
+    "You are a routing system for a multi-module assistant.\n"
+    "Your task is to rank ALL available modules by how suitable they are\n"
+    "for handling the user's input.\n\n"
+
+    "Available modules:\n"
+    "- railway_rag : authoritative, static, rule-based railway information such as\n"
+    "               laws, regulations, penalties, permissions, procedures, and\n"
+    "               official policies derived from documents.\n\n"
+
+    "- live_data_apis : dynamic or time-sensitive railway information such as\n"
+    "                   live train status, current location, delays, fares,\n"
+    "                   seat availability, PNR status, or other real-time data.\n\n"
+
+    "- general : conversational, explanatory, or contextual input such as\n"
+    "            greetings, clarifications, follow-up questions, or general\n"
+    "            explanations that do not require authoritative rules or live data.\n\n"
+
+    "- link_answer : providing official railway website links ONLY when the user explicitly asks for links, sources, websites, or external references, or when they request where to check information.\n\n"
+
+    "Rules:\n"
+    "- Rank ALL modules\n"
+    "- Relevance must be between 0.0 and 1.0\n"
+    "- Higher relevance means the module should be tried earlier\n"
+    "- Do NOT explain your reasoning\n"
+    "- Do NOT generate answers\n"
+    "- Respond ONLY in valid JSON\n\n"
+
+    "JSON format:\n"
+    "{\n"
+    '  "module_preferences": [\n'
+    '    {"module": "railway_rag", "relevance": 0.0},\n'
+    '    {"module": "live_data_apis", "relevance": 0.0},\n'
+    '    {"module": "general", "relevance": 0.0},\n'
+    '    {"module": "link_answer", "relevance": 0.0}\n'
+    "  ]\n"
+    "}"
+)
+
+VALID_MODULES = {"railway_rag", "live_data_apis", "general", "link_answer"}
+
+def route_query(query):
+    response = client.chat.completions.create(
+        model=MODEL_NAME,
+        messages=[
+            {"role": "system", "content": ROUTER_SYSTEM_PROMPT},
+            {"role": "user", "content": query}
+        ],
+        temperature=0
+    )
+    try:
+        parsed = json.loads(response.choices[0].message.content)
+        prefs = parsed["module_preferences"]
+    except Exception:
+        # Explicit router failure (no guessing)
+        return {
+            "router_failed": True,
+            "module_preferences": []
+        }
+    cleaned = []
+    seen = set()
+    for item in prefs:
+        module = item.get("module")
+        relevance = item.get("relevance")
+        if module not in VALID_MODULES:
+            continue
+        try:
+            relevance = float(relevance)
+        except Exception:
+            continue
+        relevance=max(0.0,min(relevance,1.0))
+        cleaned.append({"module": module, "relevance": relevance})   #Gives module name and relevance score in decreasing order
+        seen.add(module)
+    for m in VALID_MODULES:
+        if m not in seen:
+            cleaned.append({"module": m, "relevance": 0.0})
+    cleaned.sort(key=lambda x: x["relevance"], reverse=True)
+    print(cleaned)
+    return {
+        "router_failed": False,
+        "module_preferences": cleaned,
+    }

Chatbot/server.py

@@ -0,0 +1,55 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from fastapi.staticfiles import StaticFiles
+import os
+import traceback
+from dotenv import load_dotenv
+load_dotenv()
+from Chatbot.bot import answer_query
+
+app = FastAPI(title="Railway Assistant")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],       
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+FRONTEND_DIR = os.path.join(BASE_DIR, "frontend")
+
+class ChatRequest(BaseModel):
+    query: str
+
+class ChatResponse(BaseModel):
+    answer: str
+    confidence: str | None = None
+    source: str | None = None
+
+@app.post("/chat", response_model=ChatResponse)
+async def chat(request: ChatRequest):
+    try:
+        answer = answer_query(request.query)
+        # Defensive fallback (never return None)
+        if not answer:
+            answer = "I couldn’t process that. Please try rephrasing your question."
+    except Exception:
+        traceback.print_exc()
+        answer = "Something went wrong while processing your request."
+    return {
+        "answer": answer,
+        "confidence": None,
+        "source": None
+    }
+
+@app.get("/")
+def serve_ui():
+    return FileResponse(os.path.join(FRONTEND_DIR, "index.html"))
+
+# Serve static assets (CSS, JS, images)
+app.mount(
+    "/static",
+    StaticFiles(directory=FRONTEND_DIR),
+    name="static"
+)

app.py

@@ -0,0 +1,30 @@
+import gradio as gr
+import traceback
+from Chatbot.bot import answer_query
+
+def chat_fn(message, history):
+    try:
+        answer = answer_query(message)
+        if not answer:
+            answer = "I couldn’t process that. Please try rephrasing your question."
+    except Exception:
+        traceback.print_exc()
+        answer = "Something went wrong while processing your request."
+    history.append((message, answer))
+    return "", history
+
+with gr.Blocks(
+    theme=gr.themes.Soft(),
+    title="Railway Intelligence Assistant"
+) as demo:
+    gr.Markdown("""
+    # 🚆 Railway Intelligence Assistant  
+    **LLM-powered RAG system using FAISS, live APIs, and ChatGPT**
+    """)
+    chatbot = gr.Chatbot(height=420)
+    msg = gr.Textbox(
+        placeholder="Ask about railway rules, safety, or live train information…",
+        show_label=False
+    )
+    msg.submit(chat_fn, [msg, chatbot], [msg, chatbot])
+demo.launch()

data/chunks/live_sources_chunks.json

@@ -0,0 +1,730 @@
+[
+  {
+    "chunk_id": "https:__www.indiacode.nic.in_handle_123456789_2362_pNone_s1_c1",
+    "document_path": "https://www.indiacode.nic.in/handle/123456789/2362",
+    "doc_category": "live_source",
+    "rule_type": "law",
+    "priority": 1,
+    "authority": "Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official and updated text of the Railways Act, 1989 including amendments"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard__pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "circular",
+    "priority": 2,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Latest railway rules, circulars, policy letters and amendments issued by Railway Board"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_nget_enquiry_termsAndConditions_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/termsAndConditions",
+    "doc_category": "live_source",
+    "rule_type": "passenger_rules",
+    "priority": 3,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Latest passenger-facing rules including booking, cancellation, refund and TDR policies"
+  },
+  {
+    "chunk_id": "https:__contents.irctc.co.in_en_RefundCancellationRules.pdf_pNone_s1_c1",
+    "document_path": "https://contents.irctc.co.in/en/RefundCancellationRules.pdf",
+    "doc_category": "live_source",
+    "rule_type": "refund",
+    "priority": 3,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official refund and cancellation rules PDF maintained by IRCTC"
+  },
+  {
+    "chunk_id": "https:__railmadad.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official grievance redressal and emergency assistance portal (includes helpline 139)"
+  },
+  {
+    "chunk_id": "https:__crs.gov.in_pNone_s1_c1",
+    "document_path": "https://crs.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "safety",
+    "priority": 2,
+    "authority": "Ministry of Civil Aviation (Rail Safety)",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Railway safety rules, accident inquiries and safety manuals"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,2,287_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,2,287",
+    "doc_category": "live_source",
+    "rule_type": "passenger_amenities",
+    "priority": 4,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official passenger amenities policies and standards at stations and trains"
+  },
+  {
+    "chunk_id": "https:__rct.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://rct.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "claims_compensation",
+    "priority": 2,
+    "authority": "Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Compensation, refund and claims procedures related to railway accidents and losses"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "official_portal",
+    "priority": 1,
+    "authority": "Ministry of Railways, Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Primary official portal for Indian Railways policies, notices, tenders and updates"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "passenger_enquiry",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Passenger-facing enquiry system for schedules, routes, fares and availability"
+  },
+  {
+    "chunk_id": "https:__enquiry.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://enquiry.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_status",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official live train running status system"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_platform",
+    "priority": 0,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official e-ticketing, catering and tourism portal"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_nget_enquiry_contactus_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/contactus",
+    "doc_category": "live_source",
+    "rule_type": "customer_support",
+    "priority": 2,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Customer support, grievance and escalation information"
+  },
+  {
+    "chunk_id": "https:__railmadad.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Emergency assistance and grievance redressal platform (139)"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard__pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "policy_authority",
+    "priority": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Top policy-making authority issuing circulars, manuals and instructions"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "traffic_rules",
+    "priority": 2,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Traffic, operating and commercial circulars"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304,366_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304,366",
+    "doc_category": "live_source",
+    "rule_type": "safety_policy",
+    "priority": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Railway safety policies, manuals and advisories"
+  },
+  {
+    "chunk_id": "https:__crs.gov.in_Reports.aspx_pNone_s1_c1",
+    "document_path": "https://crs.gov.in/Reports.aspx",
+    "doc_category": "live_source",
+    "rule_type": "accident_inquiry",
+    "priority": 1,
+    "authority": "Ministry of Civil Aviation (CRS)",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Independent accident inquiry and safety inspection reports"
+  },
+  {
+    "chunk_id": "https:__prs.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://prs.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "reservation_backend",
+    "priority": 1,
+    "authority": "CRIS / Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Backend reservation infrastructure of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__cris.org.in_pNone_s1_c1",
+    "document_path": "https://cris.org.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_it",
+    "priority": 2,
+    "authority": "Ministry of Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "IT organization managing PRS, NTES, FOIS and operations software"
+  },
+  {
+    "chunk_id": "https:__www.fois.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://www.fois.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "freight_tracking",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Freight booking, tracking and policy portal"
+  },
+  {
+    "chunk_id": "https:__dfccil.com_pNone_s1_c1",
+    "document_path": "https://dfccil.com",
+    "doc_category": "live_source",
+    "rule_type": "freight_infrastructure",
+    "priority": 3,
+    "authority": "Ministry of Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Dedicated freight corridor operations and rules"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_enquiry_StaticPages_StaticEnquiry.jsp?StaticPage=station_code.html_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=station_code.html",
+    "doc_category": "live_source",
+    "rule_type": "station_reference",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official station names and station codes list"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_enquiry_StaticPages_StaticEnquiry.jsp?StaticPage=conc_Rules.html_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=conc_Rules.html",
+    "doc_category": "live_source",
+    "rule_type": "passenger_concessions",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Eligibility and rules for passenger fare concessions"
+  },
+  {
+    "chunk_id": "https:__www.rrbcdg.gov.in_pNone_s1_c1",
+    "document_path": "https://www.rrbcdg.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "recruitment",
+    "priority": 4,
+    "authority": "Railway Recruitment Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Recruitment notices, exams and results"
+  },
+  {
+    "chunk_id": "https:__rpf.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://rpf.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_security",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Railway security, passenger safety and law enforcement"
+  },
+  {
+    "chunk_id": "https:__www.ireps.gov.in_pNone_s1_c1",
+    "document_path": "https://www.ireps.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "tenders_procurement",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Official procurement and tendering platform"
+  },
+  {
+    "chunk_id": "https:__nair.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://nair.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "training",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Officer training material and manuals"
+  },
+  {
+    "chunk_id": "https:__www.iricen.gov.in_pNone_s1_c1",
+    "document_path": "https://www.iricen.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "track_engineering",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Track engineering manuals and technical guidance"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "zonal_information",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Links to all zonal railway websites for zone-specific rules"
+  },
+  {
+    "chunk_id": "https:__whereismytrain.in_pNone_s1_c1",
+    "document_path": "https://whereismytrain.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_tracking",
+    "priority": 5,
+    "authority": "Unofficial (Google)",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Widely used live train tracking app and website using crowd-sourced and offline data"
+  },
+  {
+    "chunk_id": "https:__erail.in_pNone_s1_c1",
+    "document_path": "https://erail.in",
+    "doc_category": "live_source",
+    "rule_type": "train_schedule_route",
+    "priority": 5,
+    "authority": "Unofficial",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Reliable source for train schedules, routes, stations and timetables"
+  },
+  {
+    "chunk_id": "https:__indiarailinfo.com_pNone_s1_c1",
+    "document_path": "https://indiarailinfo.com",
+    "doc_category": "live_source",
+    "rule_type": "community_updates",
+    "priority": 5,
+    "authority": "Community-driven",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Crowdsourced train running updates, delays, spotting and route discussions"
+  },
+  {
+    "chunk_id": "https:__www.railyatri.in_pNone_s1_c1",
+    "document_path": "https://www.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_live_status",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "PNR status, live train status and passenger utilities"
+  },
+  {
+    "chunk_id": "https:__www.ixigo.com_trains_pNone_s1_c1",
+    "document_path": "https://www.ixigo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "train_search",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Train search, live status and delay analytics"
+  },
+  {
+    "chunk_id": "https:__www.confirmtkt.com_pNone_s1_c1",
+    "document_path": "https://www.confirmtkt.com",
+    "doc_category": "live_source",
+    "rule_type": "seat_prediction",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Seat availability prediction and PNR insights"
+  },
+  {
+    "chunk_id": "https:__www.trainman.in_pNone_s1_c1",
+    "document_path": "https://www.trainman.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_prediction",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "PNR status prediction and train information"
+  },
+  {
+    "chunk_id": "https:__www.makemytrip.com_railways_pNone_s1_c1",
+    "document_path": "https://www.makemytrip.com/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Railway ticket booking and enquiry platform"
+  },
+  {
+    "chunk_id": "https:__www.goibibo.com_trains_pNone_s1_c1",
+    "document_path": "https://www.goibibo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Train search and booking aggregator"
+  },
+  {
+    "chunk_id": "https:__paytm.com_train-tickets_pNone_s1_c1",
+    "document_path": "https://paytm.com/train-tickets",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Train ticket booking, PNR and live status tools"
+  },
+  {
+    "chunk_id": "https:__www.redbus.in_railways_pNone_s1_c1",
+    "document_path": "https://www.redbus.in/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Rail ticket booking and enquiry features"
+  },
+  {
+    "chunk_id": "https:__railradar.railyatri.in_pNone_s1_c1",
+    "document_path": "https://railradar.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "live_map_tracking",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Map-based visualization of live train movement"
+  },
+  {
+    "chunk_id": "https:__www.openrailwaymap.org_pNone_s1_c1",
+    "document_path": "https://www.openrailwaymap.org",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "authority": "Open-source",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Detailed open-source railway infrastructure maps"
+  },
+  {
+    "chunk_id": "https:__www.mapsofindia.com_railway-map_pNone_s1_c1",
+    "document_path": "https://www.mapsofindia.com/railway-map",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Visual maps of Indian railway zones and routes"
+  },
+  {
+    "chunk_id": "https:__en.wikipedia.org_wiki_Indian_Railways_pNone_s1_c1",
+    "document_path": "https://en.wikipedia.org/wiki/Indian_Railways",
+    "doc_category": "live_source",
+    "rule_type": "encyclopedic_reference",
+    "priority": 5,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "General background, history and overview of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.wikidata.org_wiki_Q1442_pNone_s1_c1",
+    "document_path": "https://www.wikidata.org/wiki/Q1442",
+    "doc_category": "live_source",
+    "rule_type": "structured_data",
+    "priority": 5,
+    "authority": "Open knowledge",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Structured knowledge graph for Indian Railways entities"
+  },
+  {
+    "chunk_id": "https:__www.ndtv.com_topic_indian-railways_pNone_s1_c1",
+    "document_path": "https://www.ndtv.com/topic/indian-railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "News coverage related to Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.thehindu.com_topic_indian_railways__pNone_s1_c1",
+    "document_path": "https://www.thehindu.com/topic/indian_railways/",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Policy, safety and operational news on railways"
+  },
+  {
+    "chunk_id": "https:__www.livemint.com_industry_railways_pNone_s1_c1",
+    "document_path": "https://www.livemint.com/industry/railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_policy_news",
+    "priority": 5,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Economic and policy analysis of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.reddit.com_r_IndianRailways_pNone_s1_c1",
+    "document_path": "https://www.reddit.com/r/IndianRailways",
+    "doc_category": "live_source",
+    "rule_type": "community_discussion",
+    "priority": 5,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Community discussions, photos and experiences"
+  },
+  {
+    "chunk_id": "https:__www.quora.com_topic_Indian-Railways_pNone_s1_c1",
+    "document_path": "https://www.quora.com/topic/Indian-Railways",
+    "doc_category": "live_source",
+    "rule_type": "community_qna",
+    "priority": 5,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Community Q&A related to Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.youtube.com_results?search_query=indian+railways_pNone_s1_c1",
+    "document_path": "https://www.youtube.com/results?search_query=indian+railways",
+    "doc_category": "live_source",
+    "rule_type": "video_reference",
+    "priority": 5,
+    "authority": "Media / Community",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "section_index": 1,
+    "chunk_index": 1,
+    "text": "Videos on train journeys, reviews, and railway operations"
+  }
+]

data/extracted_text/live_sources.json

@@ -0,0 +1,678 @@
+[
+  {
+    "document_name": "India Code – Railways Act, 1989",
+    "document_path": "https://www.indiacode.nic.in/handle/123456789/2362",
+    "doc_category": "live_source",
+    "rule_type": "law",
+    "priority": 1,
+    "authority": "Government of India",
+    "description": "Official and updated text of the Railways Act, 1989 including amendments",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official and updated text of the Railways Act, 1989 including amendments"
+  },
+  {
+    "document_name": "Railway Board Circulars & Orders",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "circular",
+    "priority": 2,
+    "authority": "Railway Board",
+    "description": "Latest railway rules, circulars, policy letters and amendments issued by Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Latest railway rules, circulars, policy letters and amendments issued by Railway Board"
+  },
+  {
+    "document_name": "IRCTC Rules & Policies",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/termsAndConditions",
+    "doc_category": "live_source",
+    "rule_type": "passenger_rules",
+    "priority": 3,
+    "authority": "IRCTC",
+    "description": "Latest passenger-facing rules including booking, cancellation, refund and TDR policies",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Latest passenger-facing rules including booking, cancellation, refund and TDR policies"
+  },
+  {
+    "document_name": "IRCTC Refund & Cancellation Rules",
+    "document_path": "https://contents.irctc.co.in/en/RefundCancellationRules.pdf",
+    "doc_category": "live_source",
+    "rule_type": "refund",
+    "priority": 3,
+    "authority": "IRCTC",
+    "description": "Official refund and cancellation rules PDF maintained by IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official refund and cancellation rules PDF maintained by IRCTC"
+  },
+  {
+    "document_name": "RailMadad – Passenger Grievance & Emergency",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "description": "Official grievance redressal and emergency assistance portal (includes helpline 139)",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official grievance redressal and emergency assistance portal (includes helpline 139)"
+  },
+  {
+    "document_name": "Commissioner of Railway Safety",
+    "document_path": "https://crs.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "safety",
+    "priority": 2,
+    "authority": "Ministry of Civil Aviation (Rail Safety)",
+    "description": "Railway safety rules, accident inquiries and safety manuals",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Railway safety rules, accident inquiries and safety manuals"
+  },
+  {
+    "document_name": "Passenger Amenities – Indian Railways",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,2,287",
+    "doc_category": "live_source",
+    "rule_type": "passenger_amenities",
+    "priority": 4,
+    "authority": "Railway Board",
+    "description": "Official passenger amenities policies and standards at stations and trains",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official passenger amenities policies and standards at stations and trains"
+  },
+  {
+    "document_name": "Railway Claims Tribunal",
+    "document_path": "https://rct.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "claims_compensation",
+    "priority": 2,
+    "authority": "Government of India",
+    "description": "Compensation, refund and claims procedures related to railway accidents and losses",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Compensation, refund and claims procedures related to railway accidents and losses"
+  },
+  {
+    "document_name": "Indian Railways Official Website",
+    "document_path": "https://indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "official_portal",
+    "priority": 1,
+    "authority": "Ministry of Railways, Government of India",
+    "description": "Primary official portal for Indian Railways policies, notices, tenders and updates",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Primary official portal for Indian Railways policies, notices, tenders and updates"
+  },
+  {
+    "document_name": "Indian Railways Passenger Enquiry",
+    "document_path": "https://www.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "passenger_enquiry",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "description": "Passenger-facing enquiry system for schedules, routes, fares and availability",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Passenger-facing enquiry system for schedules, routes, fares and availability"
+  },
+  {
+    "document_name": "National Train Enquiry System (NTES)",
+    "document_path": "https://enquiry.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_status",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "description": "Official live train running status system",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official live train running status system"
+  },
+  {
+    "document_name": "IRCTC Official Portal",
+    "document_path": "https://www.irctc.co.in",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_platform",
+    "priority": 0,
+    "authority": "IRCTC",
+    "description": "Official e-ticketing, catering and tourism portal",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official e-ticketing, catering and tourism portal"
+  },
+  {
+    "document_name": "IRCTC Help & Customer Care",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/contactus",
+    "doc_category": "live_source",
+    "rule_type": "customer_support",
+    "priority": 2,
+    "authority": "IRCTC",
+    "description": "Customer support, grievance and escalation information",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Customer support, grievance and escalation information"
+  },
+  {
+    "document_name": "RailMadad Grievance Portal",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "authority": "Indian Railways",
+    "description": "Emergency assistance and grievance redressal platform (139)",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Emergency assistance and grievance redressal platform (139)"
+  },
+  {
+    "document_name": "Railway Board",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "policy_authority",
+    "priority": 1,
+    "authority": "Railway Board",
+    "description": "Top policy-making authority issuing circulars, manuals and instructions",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Top policy-making authority issuing circulars, manuals and instructions"
+  },
+  {
+    "document_name": "Railway Board – Traffic Directorate",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "traffic_rules",
+    "priority": 2,
+    "authority": "Railway Board",
+    "description": "Traffic, operating and commercial circulars",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Traffic, operating and commercial circulars"
+  },
+  {
+    "document_name": "Railway Board – Safety Directorate",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304,366",
+    "doc_category": "live_source",
+    "rule_type": "safety_policy",
+    "priority": 1,
+    "authority": "Railway Board",
+    "description": "Railway safety policies, manuals and advisories",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Railway safety policies, manuals and advisories"
+  },
+  {
+    "document_name": "Commissioner of Railway Safety Reports",
+    "document_path": "https://crs.gov.in/Reports.aspx",
+    "doc_category": "live_source",
+    "rule_type": "accident_inquiry",
+    "priority": 1,
+    "authority": "Ministry of Civil Aviation (CRS)",
+    "description": "Independent accident inquiry and safety inspection reports",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Independent accident inquiry and safety inspection reports"
+  },
+  {
+    "document_name": "Passenger Reservation System (PRS)",
+    "document_path": "https://prs.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "reservation_backend",
+    "priority": 1,
+    "authority": "CRIS / Indian Railways",
+    "description": "Backend reservation infrastructure of Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Backend reservation infrastructure of Indian Railways"
+  },
+  {
+    "document_name": "Centre for Railway Information Systems (CRIS)",
+    "document_path": "https://cris.org.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_it",
+    "priority": 2,
+    "authority": "Ministry of Railways",
+    "description": "IT organization managing PRS, NTES, FOIS and operations software",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "IT organization managing PRS, NTES, FOIS and operations software"
+  },
+  {
+    "document_name": "Freight Operations Information System (FOIS)",
+    "document_path": "https://www.fois.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "freight_tracking",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Freight booking, tracking and policy portal",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Freight booking, tracking and policy portal"
+  },
+  {
+    "document_name": "Dedicated Freight Corridor Corporation",
+    "document_path": "https://dfccil.com",
+    "doc_category": "live_source",
+    "rule_type": "freight_infrastructure",
+    "priority": 3,
+    "authority": "Ministry of Railways",
+    "description": "Dedicated freight corridor operations and rules",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Dedicated freight corridor operations and rules"
+  },
+  {
+    "document_name": "Station Codes Directory",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=station_code.html",
+    "doc_category": "live_source",
+    "rule_type": "station_reference",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "description": "Official station names and station codes list",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official station names and station codes list"
+  },
+  {
+    "document_name": "Passenger Concession Rules",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=conc_Rules.html",
+    "doc_category": "live_source",
+    "rule_type": "passenger_concessions",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Eligibility and rules for passenger fare concessions",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Eligibility and rules for passenger fare concessions"
+  },
+  {
+    "document_name": "Railway Recruitment Boards",
+    "document_path": "https://www.rrbcdg.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "recruitment",
+    "priority": 4,
+    "authority": "Railway Recruitment Board",
+    "description": "Recruitment notices, exams and results",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Recruitment notices, exams and results"
+  },
+  {
+    "document_name": "Railway Protection Force",
+    "document_path": "https://rpf.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_security",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Railway security, passenger safety and law enforcement",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Railway security, passenger safety and law enforcement"
+  },
+  {
+    "document_name": "Indian Railways E-Procurement",
+    "document_path": "https://www.ireps.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "tenders_procurement",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "description": "Official procurement and tendering platform",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Official procurement and tendering platform"
+  },
+  {
+    "document_name": "National Academy of Indian Railways",
+    "document_path": "https://nair.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "training",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "description": "Officer training material and manuals",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Officer training material and manuals"
+  },
+  {
+    "document_name": "IRICEN Track Engineering",
+    "document_path": "https://www.iricen.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "track_engineering",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "description": "Track engineering manuals and technical guidance",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Track engineering manuals and technical guidance"
+  },
+  {
+    "document_name": "Zonal Railways Websites Directory",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "zonal_information",
+    "priority": 4,
+    "authority": "Indian Railways",
+    "description": "Links to all zonal railway websites for zone-specific rules",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Links to all zonal railway websites for zone-specific rules"
+  },
+  {
+    "document_name": "Where Is My Train",
+    "document_path": "https://whereismytrain.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_tracking",
+    "priority": 5,
+    "authority": "Unofficial (Google)",
+    "description": "Widely used live train tracking app and website using crowd-sourced and offline data",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Widely used live train tracking app and website using crowd-sourced and offline data"
+  },
+  {
+    "document_name": "eRail",
+    "document_path": "https://erail.in",
+    "doc_category": "live_source",
+    "rule_type": "train_schedule_route",
+    "priority": 5,
+    "authority": "Unofficial",
+    "description": "Reliable source for train schedules, routes, stations and timetables",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Reliable source for train schedules, routes, stations and timetables"
+  },
+  {
+    "document_name": "India Rail Info",
+    "document_path": "https://indiarailinfo.com",
+    "doc_category": "live_source",
+    "rule_type": "community_updates",
+    "priority": 5,
+    "authority": "Community-driven",
+    "description": "Crowdsourced train running updates, delays, spotting and route discussions",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Crowdsourced train running updates, delays, spotting and route discussions"
+  },
+  {
+    "document_name": "RailYatri",
+    "document_path": "https://www.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_live_status",
+    "priority": 5,
+    "authority": "Private",
+    "description": "PNR status, live train status and passenger utilities",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "PNR status, live train status and passenger utilities"
+  },
+  {
+    "document_name": "Ixigo Trains",
+    "document_path": "https://www.ixigo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "train_search",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Train search, live status and delay analytics",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Train search, live status and delay analytics"
+  },
+  {
+    "document_name": "ConfirmTkt",
+    "document_path": "https://www.confirmtkt.com",
+    "doc_category": "live_source",
+    "rule_type": "seat_prediction",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Seat availability prediction and PNR insights",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Seat availability prediction and PNR insights"
+  },
+  {
+    "document_name": "Trainman",
+    "document_path": "https://www.trainman.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_prediction",
+    "priority": 5,
+    "authority": "Private",
+    "description": "PNR status prediction and train information",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "PNR status prediction and train information"
+  },
+  {
+    "document_name": "MakeMyTrip Railways",
+    "document_path": "https://www.makemytrip.com/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Railway ticket booking and enquiry platform",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Railway ticket booking and enquiry platform"
+  },
+  {
+    "document_name": "Goibibo Trains",
+    "document_path": "https://www.goibibo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Train search and booking aggregator",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Train search and booking aggregator"
+  },
+  {
+    "document_name": "Paytm Trains",
+    "document_path": "https://paytm.com/train-tickets",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Train ticket booking, PNR and live status tools",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Train ticket booking, PNR and live status tools"
+  },
+  {
+    "document_name": "redBus Railways",
+    "document_path": "https://www.redbus.in/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Rail ticket booking and enquiry features",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Rail ticket booking and enquiry features"
+  },
+  {
+    "document_name": "RailRadar by RailYatri",
+    "document_path": "https://railradar.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "live_map_tracking",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Map-based visualization of live train movement",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Map-based visualization of live train movement"
+  },
+  {
+    "document_name": "OpenRailwayMap",
+    "document_path": "https://www.openrailwaymap.org",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "authority": "Open-source",
+    "description": "Detailed open-source railway infrastructure maps",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Detailed open-source railway infrastructure maps"
+  },
+  {
+    "document_name": "Maps of India – Railway Map",
+    "document_path": "https://www.mapsofindia.com/railway-map",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "authority": "Private",
+    "description": "Visual maps of Indian railway zones and routes",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Visual maps of Indian railway zones and routes"
+  },
+  {
+    "document_name": "Wikipedia – Indian Railways",
+    "document_path": "https://en.wikipedia.org/wiki/Indian_Railways",
+    "doc_category": "live_source",
+    "rule_type": "encyclopedic_reference",
+    "priority": 5,
+    "authority": "Community",
+    "description": "General background, history and overview of Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "General background, history and overview of Indian Railways"
+  },
+  {
+    "document_name": "Wikidata – Indian Railways",
+    "document_path": "https://www.wikidata.org/wiki/Q1442",
+    "doc_category": "live_source",
+    "rule_type": "structured_data",
+    "priority": 5,
+    "authority": "Open knowledge",
+    "description": "Structured knowledge graph for Indian Railways entities",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Structured knowledge graph for Indian Railways entities"
+  },
+  {
+    "document_name": "NDTV – Indian Railways",
+    "document_path": "https://www.ndtv.com/topic/indian-railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "authority": "Media",
+    "description": "News coverage related to Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "News coverage related to Indian Railways"
+  },
+  {
+    "document_name": "The Hindu – Indian Railways",
+    "document_path": "https://www.thehindu.com/topic/indian_railways/",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "authority": "Media",
+    "description": "Policy, safety and operational news on railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Policy, safety and operational news on railways"
+  },
+  {
+    "document_name": "LiveMint – Railways",
+    "document_path": "https://www.livemint.com/industry/railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_policy_news",
+    "priority": 5,
+    "authority": "Media",
+    "description": "Economic and policy analysis of Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Economic and policy analysis of Indian Railways"
+  },
+  {
+    "document_name": "Reddit – r/IndianRailways",
+    "document_path": "https://www.reddit.com/r/IndianRailways",
+    "doc_category": "live_source",
+    "rule_type": "community_discussion",
+    "priority": 5,
+    "authority": "Community",
+    "description": "Community discussions, photos and experiences",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Community discussions, photos and experiences"
+  },
+  {
+    "document_name": "Quora – Indian Railways",
+    "document_path": "https://www.quora.com/topic/Indian-Railways",
+    "doc_category": "live_source",
+    "rule_type": "community_qna",
+    "priority": 5,
+    "authority": "Community",
+    "description": "Community Q&A related to Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Community Q&A related to Indian Railways"
+  },
+  {
+    "document_name": "YouTube – Indian Railways Content",
+    "document_path": "https://www.youtube.com/results?search_query=indian+railways",
+    "doc_category": "live_source",
+    "rule_type": "video_reference",
+    "priority": 5,
+    "authority": "Media / Community",
+    "description": "Videos on train journeys, reviews, and railway operations",
+    "is_static": false,
+    "effective_year": null,
+    "page_number": null,
+    "text": "Videos on train journeys, reviews, and railway operations"
+  }
+]

data/raw_docs/live_sources/live_sources.json

@@ -0,0 +1,471 @@
+[
+  {
+    "id": "india_code_railways_act",
+    "name": "India Code – Railways Act, 1989",
+    "url": "https://www.indiacode.nic.in/handle/123456789/2362",
+    "authority": "Government of India",
+    "rule_type": "law",
+    "priority": 1,
+    "description": "Official and updated text of the Railways Act, 1989 including amendments"
+  },
+  {
+    "id": "railway_board_circulars",
+    "name": "Railway Board Circulars & Orders",
+    "url": "https://indianrailways.gov.in/railwayboard/",
+    "authority": "Railway Board",
+    "rule_type": "circular",
+    "priority": 2,
+    "description": "Latest railway rules, circulars, policy letters and amendments issued by Railway Board"
+  },
+  {
+    "id": "irctc_rules",
+    "name": "IRCTC Rules & Policies",
+    "url": "https://www.irctc.co.in/nget/enquiry/termsAndConditions",
+    "authority": "IRCTC",
+    "rule_type": "passenger_rules",
+    "priority": 3,
+    "description": "Latest passenger-facing rules including booking, cancellation, refund and TDR policies"
+  },
+  {
+    "id": "irctc_refund_rules",
+    "name": "IRCTC Refund & Cancellation Rules",
+    "url": "https://contents.irctc.co.in/en/RefundCancellationRules.pdf",
+    "authority": "IRCTC",
+    "rule_type": "refund",
+    "priority": 3,
+    "description": "Official refund and cancellation rules PDF maintained by IRCTC"
+  },
+  {
+    "id": "railmadad",
+    "name": "RailMadad – Passenger Grievance & Emergency",
+    "url": "https://railmadad.indianrailways.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "description": "Official grievance redressal and emergency assistance portal (includes helpline 139)"
+  },
+  {
+    "id": "crs_safety",
+    "name": "Commissioner of Railway Safety",
+    "url": "https://crs.gov.in",
+    "authority": "Ministry of Civil Aviation (Rail Safety)",
+    "rule_type": "safety",
+    "priority": 2,
+    "description": "Railway safety rules, accident inquiries and safety manuals"
+  },
+  {
+    "id": "indian_railways_passenger_amenities",
+    "name": "Passenger Amenities – Indian Railways",
+    "url": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,2,287",
+    "authority": "Railway Board",
+    "rule_type": "passenger_amenities",
+    "priority": 4,
+    "description": "Official passenger amenities policies and standards at stations and trains"
+  },
+  {
+    "id": "railway_claims_tribunal",
+    "name": "Railway Claims Tribunal",
+    "url": "https://rct.indianrailways.gov.in",
+    "authority": "Government of India",
+    "rule_type": "claims_compensation",
+    "priority": 2,
+    "description": "Compensation, refund and claims procedures related to railway accidents and losses"
+  },
+  {
+    "id": "indian_railways_main",
+    "name": "Indian Railways Official Website",
+    "url": "https://indianrailways.gov.in",
+    "authority": "Ministry of Railways, Government of India",
+    "rule_type": "official_portal",
+    "priority": 1,
+    "description": "Primary official portal for Indian Railways policies, notices, tenders and updates"
+  },
+  {
+    "id": "indian_rail_enquiry",
+    "name": "Indian Railways Passenger Enquiry",
+    "url": "https://www.indianrail.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "passenger_enquiry",
+    "priority": 0,
+    "description": "Passenger-facing enquiry system for schedules, routes, fares and availability"
+  },
+  {
+    "id": "ntes_live_status",
+    "name": "National Train Enquiry System (NTES)",
+    "url": "https://enquiry.indianrail.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "live_train_status",
+    "priority": 0,
+    "description": "Official live train running status system"
+  },
+  {
+    "id": "irctc_home",
+    "name": "IRCTC Official Portal",
+    "url": "https://www.irctc.co.in",
+    "authority": "IRCTC",
+    "rule_type": "ticketing_platform",
+    "priority": 0,
+    "description": "Official e-ticketing, catering and tourism portal"
+  },
+  {
+    "id": "irctc_helpdesk",
+    "name": "IRCTC Help & Customer Care",
+    "url": "https://www.irctc.co.in/nget/enquiry/contactus",
+    "authority": "IRCTC",
+    "rule_type": "customer_support",
+    "priority": 2,
+    "description": "Customer support, grievance and escalation information"
+  },
+  {
+    "id": "railmadad_portal",
+    "name": "RailMadad Grievance Portal",
+    "url": "https://railmadad.indianrailways.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "description": "Emergency assistance and grievance redressal platform (139)"
+  },
+  {
+    "id": "railway_board_main",
+    "name": "Railway Board",
+    "url": "https://indianrailways.gov.in/railwayboard/",
+    "authority": "Railway Board",
+    "rule_type": "policy_authority",
+    "priority": 1,
+    "description": "Top policy-making authority issuing circulars, manuals and instructions"
+  },
+  {
+    "id": "railway_board_traffic",
+    "name": "Railway Board – Traffic Directorate",
+    "url": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "authority": "Railway Board",
+    "rule_type": "traffic_rules",
+    "priority": 2,
+    "description": "Traffic, operating and commercial circulars"
+  },
+  {
+    "id": "railway_board_safety",
+    "name": "Railway Board – Safety Directorate",
+    "url": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304,366",
+    "authority": "Railway Board",
+    "rule_type": "safety_policy",
+    "priority": 1,
+    "description": "Railway safety policies, manuals and advisories"
+  },
+  {
+    "id": "crs_reports",
+    "name": "Commissioner of Railway Safety Reports",
+    "url": "https://crs.gov.in/Reports.aspx",
+    "authority": "Ministry of Civil Aviation (CRS)",
+    "rule_type": "accident_inquiry",
+    "priority": 1,
+    "description": "Independent accident inquiry and safety inspection reports"
+  },
+  {
+    "id": "prs_system",
+    "name": "Passenger Reservation System (PRS)",
+    "url": "https://prs.indianrail.gov.in",
+    "authority": "CRIS / Indian Railways",
+    "rule_type": "reservation_backend",
+    "priority": 1,
+    "description": "Backend reservation infrastructure of Indian Railways"
+  },
+  {
+    "id": "cris_org",
+    "name": "Centre for Railway Information Systems (CRIS)",
+    "url": "https://cris.org.in",
+    "authority": "Ministry of Railways",
+    "rule_type": "railway_it",
+    "priority": 2,
+    "description": "IT organization managing PRS, NTES, FOIS and operations software"
+  },
+  {
+    "id": "fois_freight",
+    "name": "Freight Operations Information System (FOIS)",
+    "url": "https://www.fois.indianrail.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "freight_tracking",
+    "priority": 1,
+    "description": "Freight booking, tracking and policy portal"
+  },
+  {
+    "id": "dfccil_portal",
+    "name": "Dedicated Freight Corridor Corporation",
+    "url": "https://dfccil.com",
+    "authority": "Ministry of Railways",
+    "rule_type": "freight_infrastructure",
+    "priority": 3,
+    "description": "Dedicated freight corridor operations and rules"
+  },
+  {
+    "id": "station_codes",
+    "name": "Station Codes Directory",
+    "url": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=station_code.html",
+    "authority": "Indian Railways",
+    "rule_type": "station_reference",
+    "priority": 3,
+    "description": "Official station names and station codes list"
+  },
+  {
+    "id": "concession_rules",
+    "name": "Passenger Concession Rules",
+    "url": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=conc_Rules.html",
+    "authority": "Indian Railways",
+    "rule_type": "passenger_concessions",
+    "priority": 2,
+    "description": "Eligibility and rules for passenger fare concessions"
+  },
+  {
+    "id": "railway_recruitment",
+    "name": "Railway Recruitment Boards",
+    "url": "https://www.rrbcdg.gov.in",
+    "authority": "Railway Recruitment Board",
+    "rule_type": "recruitment",
+    "priority": 4,
+    "description": "Recruitment notices, exams and results"
+  },
+  {
+    "id": "rpf_portal",
+    "name": "Railway Protection Force",
+    "url": "https://rpf.indianrailways.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "railway_security",
+    "priority": 2,
+    "description": "Railway security, passenger safety and law enforcement"
+  },
+  {
+    "id": "indian_railways_tenders",
+    "name": "Indian Railways E-Procurement",
+    "url": "https://www.ireps.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "tenders_procurement",
+    "priority": 4,
+    "description": "Official procurement and tendering platform"
+  },
+  {
+    "id": "nair_training",
+    "name": "National Academy of Indian Railways",
+    "url": "https://nair.indianrailways.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "training",
+    "priority": 4,
+    "description": "Officer training material and manuals"
+  },
+  {
+    "id": "iricen_track",
+    "name": "IRICEN Track Engineering",
+    "url": "https://www.iricen.gov.in",
+    "authority": "Indian Railways",
+    "rule_type": "track_engineering",
+    "priority": 3,
+    "description": "Track engineering manuals and technical guidance"
+  },
+  {
+    "id": "zonal_railways",
+    "name": "Zonal Railways Websites Directory",
+    "url": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "authority": "Indian Railways",
+    "rule_type": "zonal_information",
+    "priority": 4,
+    "description": "Links to all zonal railway websites for zone-specific rules"
+  },
+  {
+    "id": "where_is_my_train",
+    "name": "Where Is My Train",
+    "url": "https://whereismytrain.in",
+    "authority": "Unofficial (Google)",
+    "rule_type": "live_train_tracking",
+    "priority": 5,
+    "description": "Widely used live train tracking app and website using crowd-sourced and offline data"
+  },
+  {
+    "id": "erail",
+    "name": "eRail",
+    "url": "https://erail.in",
+    "authority": "Unofficial",
+    "rule_type": "train_schedule_route",
+    "priority": 5,
+    "description": "Reliable source for train schedules, routes, stations and timetables"
+  },
+  {
+    "id": "indiarailinfo",
+    "name": "India Rail Info",
+    "url": "https://indiarailinfo.com",
+    "authority": "Community-driven",
+    "rule_type": "community_updates",
+    "priority": 5,
+    "description": "Crowdsourced train running updates, delays, spotting and route discussions"
+  },
+  {
+    "id": "railyatri",
+    "name": "RailYatri",
+    "url": "https://www.railyatri.in",
+    "authority": "Private",
+    "rule_type": "pnr_live_status",
+    "priority": 5,
+    "description": "PNR status, live train status and passenger utilities"
+  },
+  {
+    "id": "ixigo_trains",
+    "name": "Ixigo Trains",
+    "url": "https://www.ixigo.com/trains",
+    "authority": "Private",
+    "rule_type": "train_search",
+    "priority": 5,
+    "description": "Train search, live status and delay analytics"
+  },
+  {
+    "id": "confirmtkt",
+    "name": "ConfirmTkt",
+    "url": "https://www.confirmtkt.com",
+    "authority": "Private",
+    "rule_type": "seat_prediction",
+    "priority": 5,
+    "description": "Seat availability prediction and PNR insights"
+  },
+  {
+    "id": "trainman",
+    "name": "Trainman",
+    "url": "https://www.trainman.in",
+    "authority": "Private",
+    "rule_type": "pnr_prediction",
+    "priority": 5,
+    "description": "PNR status prediction and train information"
+  },
+  {
+    "id": "makemytrip_trains",
+    "name": "MakeMyTrip Railways",
+    "url": "https://www.makemytrip.com/railways",
+    "authority": "Private",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "description": "Railway ticket booking and enquiry platform"
+  },
+  {
+    "id": "goibibo_trains",
+    "name": "Goibibo Trains",
+    "url": "https://www.goibibo.com/trains",
+    "authority": "Private",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "description": "Train search and booking aggregator"
+  },
+  {
+    "id": "paytm_trains",
+    "name": "Paytm Trains",
+    "url": "https://paytm.com/train-tickets",
+    "authority": "Private",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "description": "Train ticket booking, PNR and live status tools"
+  },
+  {
+    "id": "redbus_trains",
+    "name": "redBus Railways",
+    "url": "https://www.redbus.in/railways",
+    "authority": "Private",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "description": "Rail ticket booking and enquiry features"
+  },
+  {
+    "id": "railradar_railyatri",
+    "name": "RailRadar by RailYatri",
+    "url": "https://railradar.railyatri.in",
+    "authority": "Private",
+    "rule_type": "live_map_tracking",
+    "priority": 5,
+    "description": "Map-based visualization of live train movement"
+  },
+  {
+    "id": "openrailwaymap",
+    "name": "OpenRailwayMap",
+    "url": "https://www.openrailwaymap.org",
+    "authority": "Open-source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "description": "Detailed open-source railway infrastructure maps"
+  },
+  {
+    "id": "mapsofindia_railways",
+    "name": "Maps of India – Railway Map",
+    "url": "https://www.mapsofindia.com/railway-map",
+    "authority": "Private",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "description": "Visual maps of Indian railway zones and routes"
+  },
+  {
+    "id": "wikipedia_indian_railways",
+    "name": "Wikipedia – Indian Railways",
+    "url": "https://en.wikipedia.org/wiki/Indian_Railways",
+    "authority": "Community",
+    "rule_type": "encyclopedic_reference",
+    "priority": 5,
+    "description": "General background, history and overview of Indian Railways"
+  },
+  {
+    "id": "wikidata_indian_railways",
+    "name": "Wikidata – Indian Railways",
+    "url": "https://www.wikidata.org/wiki/Q1442",
+    "authority": "Open knowledge",
+    "rule_type": "structured_data",
+    "priority": 5,
+    "description": "Structured knowledge graph for Indian Railways entities"
+  },
+  {
+    "id": "ndtv_railways",
+    "name": "NDTV – Indian Railways",
+    "url": "https://www.ndtv.com/topic/indian-railways",
+    "authority": "Media",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "description": "News coverage related to Indian Railways"
+  },
+  {
+    "id": "the_hindu_railways",
+    "name": "The Hindu – Indian Railways",
+    "url": "https://www.thehindu.com/topic/indian_railways/",
+    "authority": "Media",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "description": "Policy, safety and operational news on railways"
+  },
+  {
+    "id": "livemint_railways",
+    "name": "LiveMint – Railways",
+    "url": "https://www.livemint.com/industry/railways",
+    "authority": "Media",
+    "rule_type": "railway_policy_news",
+    "priority": 5,
+    "description": "Economic and policy analysis of Indian Railways"
+  },
+  {
+    "id": "reddit_indianrailways",
+    "name": "Reddit – r/IndianRailways",
+    "url": "https://www.reddit.com/r/IndianRailways",
+    "authority": "Community",
+    "rule_type": "community_discussion",
+    "priority": 5,
+    "description": "Community discussions, photos and experiences"
+  },
+  {
+    "id": "quora_indian_railways",
+    "name": "Quora – Indian Railways",
+    "url": "https://www.quora.com/topic/Indian-Railways",
+    "authority": "Community",
+    "rule_type": "community_qna",
+    "priority": 5,
+    "description": "Community Q&A related to Indian Railways"
+  },
+  {
+    "id": "youtube_indian_railways",
+    "name": "YouTube – Indian Railways Content",
+    "url": "https://www.youtube.com/results?search_query=indian+railways",
+    "authority": "Media / Community",
+    "rule_type": "video_reference",
+    "priority": 5,
+    "description": "Videos on train journeys, reviews, and railway operations"
+  }
+]
+

data/raw_docs/meta_data.json

@@ -0,0 +1,283 @@
+{
+  "core_docs/04_Safety_210815.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "safety",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "description": "Safety instructions and procedures issued by Indian Railways",
+    "is_static": true
+  },
+
+  "core_docs/1416206396757-Reservation Rules.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "reservation",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Core reservation rules governing passenger ticket booking and allocation",
+    "is_static": true
+  },
+
+  "core_docs/AA1987___54railway.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "claims_compensation",
+    "priority": 1,
+    "authority": "Government of India",
+    "description": "Railway Claims Tribunal Act, 1987 governing compensation and claims",
+    "is_static": true
+  },
+
+  "core_docs/Accident Protocol or Manual (Indian Railways).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "safety",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Official accident handling and investigation procedures",
+    "is_static": true
+  },
+
+  "core_docs/Block Working Manual (Eastern Railway).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Rules and procedures for block working on Eastern Railway",
+    "is_static": true
+  },
+
+  "core_docs/Dedicated Freight Corridor – General Rules.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 1,
+    "authority": "DFCCIL / Indian Railways",
+    "description": "General rules governing operations on Dedicated Freight Corridors",
+    "is_static": true
+  },
+
+  "core_docs/General Rules (South Central Railway G&SR PDF).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "General and Subsidiary Rules applicable to South Central Railway",
+    "is_static": true
+  },
+
+  "core_docs/Indian Railways – General & Subsidiary Rules (North Western Railway).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "General and Subsidiary Rules applicable to North Western Railway",
+    "is_static": true
+  },
+
+  "core_docs/Indian Railways – General & Subsidiary Rules(Northern Railway, 2025).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Latest General and Subsidiary Rules for Northern Railway (2025 edition)",
+    "is_static": true
+  },
+
+  "core_docs/Indian Railways – General rules (Southern Railway 1976 GRS).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Historical General Rules (1976) for Southern Railway",
+    "is_static": true
+  },
+
+  "core_docs/Indian Railways – Reservation Rules (English).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "reservation",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Authoritative English version of Indian Railways Reservation Rules",
+    "is_static": true
+  },
+
+  "core_docs/operating manual-traffic.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Operating manual governing train traffic and movement",
+    "is_static": true
+  },
+
+  "core_docs/Railway Servants (Pass) Rules, 1986.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "staff_rules",
+    "priority": 1,
+    "authority": "Government of India",
+    "description": "Rules governing railway staff travel privileges and passes",
+    "is_static": true
+  },
+
+  "core_docs/Railway-Notices-of-and-Inquiries-into-Accidents-Rules-1998.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "safety",
+    "priority": 1,
+    "authority": "Indian Railways",
+    "description": "Statutory rules for railway accident notices and inquiries",
+    "is_static": true
+  },
+
+  "core_docs/Railways_Act_1989.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "law",
+    "priority": 1,
+    "authority": "Government of India",
+    "description": "Primary legislation governing Indian Railways",
+    "is_static": true
+  },
+
+  "core_docs/RulesAndRegulations-2013.pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "passenger_rules",
+    "priority": 2,
+    "authority": "IRCTC / Indian Railways",
+    "description": "IRCTC rules and regulations for online ticketing (base framework)",
+    "is_static": true
+  },
+
+  "core_docs/Station Working Rules – Rupra Road (East Coast Railway).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Station Working Rules for Rupra Road station",
+    "is_static": true
+  },
+
+  "core_docs/Station Working Rules Chapter from NER (PDF list section).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "operations",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Station Working Rules chapter reference from North Eastern Railway",
+    "is_static": true
+  },
+
+  "core_docs/The Railways Act, 1989 (Official PDF).pdf": {
+    "doc_category": "core_docs",
+    "rule_type": "law",
+    "priority": 1,
+    "authority": "Government of India",
+    "description": "Official consolidated PDF of the Railways Act, 1989",
+    "is_static": true
+  },
+
+  "circulars/2013/REFUND RULES wef 1-Jul-13.pdf": {
+    "doc_category": "circular",
+    "rule_type": "refund",
+    "priority": 2,
+    "authority": "IRCTC / Railway Board",
+    "description": "Refund rules effective from 1 July 2013",
+    "is_static": false,
+    "effective_year": 2013
+  },
+
+  "circulars/2015/CancellationRulesforIRCTCTrain.pdf": {
+    "doc_category": "circular",
+    "rule_type": "cancellation",
+    "priority": 2,
+    "authority": "IRCTC",
+    "description": "Cancellation rules applicable to IRCTC train tickets",
+    "is_static": false,
+    "effective_year": 2015
+  },
+
+  "circulars/2023/2023_11_23 Final_after_CORRIGENDUM_Signal dte amendment.pdf": {
+    "doc_category": "circular",
+    "rule_type": "signalling_rules",
+    "priority": 2,
+    "authority": "Railway Board (Signal Directorate)",
+    "description": "Corrigendum amending signalling-related operational rules",
+    "is_static": false,
+    "effective_year": 2023
+  },
+
+  "circulars/Concession Rules (General Rules for Concession).pdf": {
+    "doc_category": "circular",
+    "rule_type": "passenger_concessions",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "General rules governing passenger fare concessions",
+    "is_static": false
+  },
+
+  "circulars/IRCTC – Rules & Regulations (For Agents).pdf": {
+    "doc_category": "circular",
+    "rule_type": "agent_rules",
+    "priority": 2,
+    "authority": "IRCTC",
+    "description": "Rules and regulations applicable to IRCTC agents",
+    "is_static": false
+  },
+
+  "circulars/IRCTC Tatkal Booking FAQ.pdf": {
+    "doc_category": "circular",
+    "rule_type": "tatkal",
+    "priority": 3,
+    "authority": "IRCTC",
+    "description": "Frequently asked questions related to Tatkal ticket booking",
+    "is_static": false
+  },
+
+  "circulars/IRCTC User Guide_ Tatkal Ticket Booking (IRCTC).pdf": {
+    "doc_category": "circular",
+    "rule_type": "tatkal",
+    "priority": 3,
+    "authority": "IRCTC",
+    "description": "User guide explaining Tatkal ticket booking process",
+    "is_static": false
+  },
+
+  "circulars/Railway Establishment Rules.pdf": {
+    "doc_category": "circular",
+    "rule_type": "staff_rules",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Establishment rules governing railway staff service matters",
+    "is_static": false
+  },
+
+  "circulars/Railway Track Machine Manual.pdf": {
+    "doc_category": "circular",
+    "rule_type": "engineering",
+    "priority": 3,
+    "authority": "Indian Railways",
+    "description": "Manual for operation and maintenance of railway track machines",
+    "is_static": false
+  },
+
+  "circulars/Tatkal Scheme (Indian Railways Board PDF).pdf": {
+    "doc_category": "circular",
+    "rule_type": "tatkal",
+    "priority": 2,
+    "authority": "Railway Board",
+    "description": "Official Tatkal scheme issued by Railway Board",
+    "is_static": false
+  },
+
+  "circulars/Ticket Checking Manual (West Central Railway).pdf": {
+    "doc_category": "circular",
+    "rule_type": "ticket_checking",
+    "priority": 2,
+    "authority": "Indian Railways",
+    "description": "Procedures and powers of ticket checking staff",
+    "is_static": false
+  },
+
+  "circulars/Unreserved Ticketing System Chapter (CAG PDF).pdf": {
+    "doc_category": "circular",
+    "rule_type": "unreserved_ticketing",
+    "priority": 3,
+    "authority": "Comptroller and Auditor General of India",
+    "description": "Audit chapter on Unreserved Ticketing System",
+    "is_static": false
+  }
+}

data/raw_docs/train_station/EXP-TRAINS.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd26de1acda7711b2c664f7f8b6ae0091f6b8792db086245f849f026a427e41c
+size 17507111

data/raw_docs/train_station/PASS-TRAINS.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a8f9107dbd004f043094fce79fca06bc3008b728bcaf1376311e5f2735d4eb6
+size 21278197

data/vector_store/live_metadata.json

@@ -0,0 +1,678 @@
+[
+  {
+    "chunk_id": "https:__www.indiacode.nic.in_handle_123456789_2362_pNone_s1_c1",
+    "document_path": "https://www.indiacode.nic.in/handle/123456789/2362",
+    "doc_category": "live_source",
+    "rule_type": "law",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official and updated text of the Railways Act, 1989 including amendments"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard__pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "circular",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Latest railway rules, circulars, policy letters and amendments issued by Railway Board"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_nget_enquiry_termsAndConditions_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/termsAndConditions",
+    "doc_category": "live_source",
+    "rule_type": "passenger_rules",
+    "priority": 3,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Latest passenger-facing rules including booking, cancellation, refund and TDR policies"
+  },
+  {
+    "chunk_id": "https:__contents.irctc.co.in_en_RefundCancellationRules.pdf_pNone_s1_c1",
+    "document_path": "https://contents.irctc.co.in/en/RefundCancellationRules.pdf",
+    "doc_category": "live_source",
+    "rule_type": "refund",
+    "priority": 3,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official refund and cancellation rules PDF maintained by IRCTC"
+  },
+  {
+    "chunk_id": "https:__railmadad.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official grievance redressal and emergency assistance portal (includes helpline 139)"
+  },
+  {
+    "chunk_id": "https:__crs.gov.in_pNone_s1_c1",
+    "document_path": "https://crs.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "safety",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Ministry of Civil Aviation (Rail Safety)",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Railway safety rules, accident inquiries and safety manuals"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,2,287_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,2,287",
+    "doc_category": "live_source",
+    "rule_type": "passenger_amenities",
+    "priority": 4,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official passenger amenities policies and standards at stations and trains"
+  },
+  {
+    "chunk_id": "https:__rct.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://rct.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "claims_compensation",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Compensation, refund and claims procedures related to railway accidents and losses"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "official_portal",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Ministry of Railways, Government of India",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Primary official portal for Indian Railways policies, notices, tenders and updates"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "passenger_enquiry",
+    "priority": 0,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Passenger-facing enquiry system for schedules, routes, fares and availability"
+  },
+  {
+    "chunk_id": "https:__enquiry.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://enquiry.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_status",
+    "priority": 0,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official live train running status system"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_platform",
+    "priority": 0,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official e-ticketing, catering and tourism portal"
+  },
+  {
+    "chunk_id": "https:__www.irctc.co.in_nget_enquiry_contactus_pNone_s1_c1",
+    "document_path": "https://www.irctc.co.in/nget/enquiry/contactus",
+    "doc_category": "live_source",
+    "rule_type": "customer_support",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "IRCTC",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Customer support, grievance and escalation information"
+  },
+  {
+    "chunk_id": "https:__railmadad.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://railmadad.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "grievance_emergency",
+    "priority": 0,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Emergency assistance and grievance redressal platform (139)"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard__pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/",
+    "doc_category": "live_source",
+    "rule_type": "policy_authority",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Top policy-making authority issuing circulars, manuals and instructions"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "traffic_rules",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Traffic, operating and commercial circulars"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304,366_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304,366",
+    "doc_category": "live_source",
+    "rule_type": "safety_policy",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Railway safety policies, manuals and advisories"
+  },
+  {
+    "chunk_id": "https:__crs.gov.in_Reports.aspx_pNone_s1_c1",
+    "document_path": "https://crs.gov.in/Reports.aspx",
+    "doc_category": "live_source",
+    "rule_type": "accident_inquiry",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Ministry of Civil Aviation (CRS)",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Independent accident inquiry and safety inspection reports"
+  },
+  {
+    "chunk_id": "https:__prs.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://prs.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "reservation_backend",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "CRIS / Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Backend reservation infrastructure of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__cris.org.in_pNone_s1_c1",
+    "document_path": "https://cris.org.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_it",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Ministry of Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "IT organization managing PRS, NTES, FOIS and operations software"
+  },
+  {
+    "chunk_id": "https:__www.fois.indianrail.gov.in_pNone_s1_c1",
+    "document_path": "https://www.fois.indianrail.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "freight_tracking",
+    "priority": 1,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Freight booking, tracking and policy portal"
+  },
+  {
+    "chunk_id": "https:__dfccil.com_pNone_s1_c1",
+    "document_path": "https://dfccil.com",
+    "doc_category": "live_source",
+    "rule_type": "freight_infrastructure",
+    "priority": 3,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Ministry of Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Dedicated freight corridor operations and rules"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_enquiry_StaticPages_StaticEnquiry.jsp?StaticPage=station_code.html_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=station_code.html",
+    "doc_category": "live_source",
+    "rule_type": "station_reference",
+    "priority": 3,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official station names and station codes list"
+  },
+  {
+    "chunk_id": "https:__www.indianrail.gov.in_enquiry_StaticPages_StaticEnquiry.jsp?StaticPage=conc_Rules.html_pNone_s1_c1",
+    "document_path": "https://www.indianrail.gov.in/enquiry/StaticPages/StaticEnquiry.jsp?StaticPage=conc_Rules.html",
+    "doc_category": "live_source",
+    "rule_type": "passenger_concessions",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Eligibility and rules for passenger fare concessions"
+  },
+  {
+    "chunk_id": "https:__www.rrbcdg.gov.in_pNone_s1_c1",
+    "document_path": "https://www.rrbcdg.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "recruitment",
+    "priority": 4,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Railway Recruitment Board",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Recruitment notices, exams and results"
+  },
+  {
+    "chunk_id": "https:__rpf.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://rpf.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "railway_security",
+    "priority": 2,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Railway security, passenger safety and law enforcement"
+  },
+  {
+    "chunk_id": "https:__www.ireps.gov.in_pNone_s1_c1",
+    "document_path": "https://www.ireps.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "tenders_procurement",
+    "priority": 4,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Official procurement and tendering platform"
+  },
+  {
+    "chunk_id": "https:__nair.indianrailways.gov.in_pNone_s1_c1",
+    "document_path": "https://nair.indianrailways.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "training",
+    "priority": 4,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Officer training material and manuals"
+  },
+  {
+    "chunk_id": "https:__www.iricen.gov.in_pNone_s1_c1",
+    "document_path": "https://www.iricen.gov.in",
+    "doc_category": "live_source",
+    "rule_type": "track_engineering",
+    "priority": 3,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Track engineering manuals and technical guidance"
+  },
+  {
+    "chunk_id": "https:__indianrailways.gov.in_railwayboard_view_section.jsp?id=0,1,304_pNone_s1_c1",
+    "document_path": "https://indianrailways.gov.in/railwayboard/view_section.jsp?id=0,1,304",
+    "doc_category": "live_source",
+    "rule_type": "zonal_information",
+    "priority": 4,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Indian Railways",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Links to all zonal railway websites for zone-specific rules"
+  },
+  {
+    "chunk_id": "https:__whereismytrain.in_pNone_s1_c1",
+    "document_path": "https://whereismytrain.in",
+    "doc_category": "live_source",
+    "rule_type": "live_train_tracking",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Unofficial (Google)",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Widely used live train tracking app and website using crowd-sourced and offline data"
+  },
+  {
+    "chunk_id": "https:__erail.in_pNone_s1_c1",
+    "document_path": "https://erail.in",
+    "doc_category": "live_source",
+    "rule_type": "train_schedule_route",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Unofficial",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Reliable source for train schedules, routes, stations and timetables"
+  },
+  {
+    "chunk_id": "https:__indiarailinfo.com_pNone_s1_c1",
+    "document_path": "https://indiarailinfo.com",
+    "doc_category": "live_source",
+    "rule_type": "community_updates",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Community-driven",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Crowdsourced train running updates, delays, spotting and route discussions"
+  },
+  {
+    "chunk_id": "https:__www.railyatri.in_pNone_s1_c1",
+    "document_path": "https://www.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_live_status",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "PNR status, live train status and passenger utilities"
+  },
+  {
+    "chunk_id": "https:__www.ixigo.com_trains_pNone_s1_c1",
+    "document_path": "https://www.ixigo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "train_search",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Train search, live status and delay analytics"
+  },
+  {
+    "chunk_id": "https:__www.confirmtkt.com_pNone_s1_c1",
+    "document_path": "https://www.confirmtkt.com",
+    "doc_category": "live_source",
+    "rule_type": "seat_prediction",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Seat availability prediction and PNR insights"
+  },
+  {
+    "chunk_id": "https:__www.trainman.in_pNone_s1_c1",
+    "document_path": "https://www.trainman.in",
+    "doc_category": "live_source",
+    "rule_type": "pnr_prediction",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "PNR status prediction and train information"
+  },
+  {
+    "chunk_id": "https:__www.makemytrip.com_railways_pNone_s1_c1",
+    "document_path": "https://www.makemytrip.com/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Railway ticket booking and enquiry platform"
+  },
+  {
+    "chunk_id": "https:__www.goibibo.com_trains_pNone_s1_c1",
+    "document_path": "https://www.goibibo.com/trains",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Train search and booking aggregator"
+  },
+  {
+    "chunk_id": "https:__paytm.com_train-tickets_pNone_s1_c1",
+    "document_path": "https://paytm.com/train-tickets",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Train ticket booking, PNR and live status tools"
+  },
+  {
+    "chunk_id": "https:__www.redbus.in_railways_pNone_s1_c1",
+    "document_path": "https://www.redbus.in/railways",
+    "doc_category": "live_source",
+    "rule_type": "ticketing_aggregator",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Rail ticket booking and enquiry features"
+  },
+  {
+    "chunk_id": "https:__railradar.railyatri.in_pNone_s1_c1",
+    "document_path": "https://railradar.railyatri.in",
+    "doc_category": "live_source",
+    "rule_type": "live_map_tracking",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Map-based visualization of live train movement"
+  },
+  {
+    "chunk_id": "https:__www.openrailwaymap.org_pNone_s1_c1",
+    "document_path": "https://www.openrailwaymap.org",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Open-source",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Detailed open-source railway infrastructure maps"
+  },
+  {
+    "chunk_id": "https:__www.mapsofindia.com_railway-map_pNone_s1_c1",
+    "document_path": "https://www.mapsofindia.com/railway-map",
+    "doc_category": "live_source",
+    "rule_type": "railway_map",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Private",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Visual maps of Indian railway zones and routes"
+  },
+  {
+    "chunk_id": "https:__en.wikipedia.org_wiki_Indian_Railways_pNone_s1_c1",
+    "document_path": "https://en.wikipedia.org/wiki/Indian_Railways",
+    "doc_category": "live_source",
+    "rule_type": "encyclopedic_reference",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "text": "General background, history and overview of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.wikidata.org_wiki_Q1442_pNone_s1_c1",
+    "document_path": "https://www.wikidata.org/wiki/Q1442",
+    "doc_category": "live_source",
+    "rule_type": "structured_data",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Open knowledge",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Structured knowledge graph for Indian Railways entities"
+  },
+  {
+    "chunk_id": "https:__www.ndtv.com_topic_indian-railways_pNone_s1_c1",
+    "document_path": "https://www.ndtv.com/topic/indian-railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "text": "News coverage related to Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.thehindu.com_topic_indian_railways__pNone_s1_c1",
+    "document_path": "https://www.thehindu.com/topic/indian_railways/",
+    "doc_category": "live_source",
+    "rule_type": "railway_news",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Policy, safety and operational news on railways"
+  },
+  {
+    "chunk_id": "https:__www.livemint.com_industry_railways_pNone_s1_c1",
+    "document_path": "https://www.livemint.com/industry/railways",
+    "doc_category": "live_source",
+    "rule_type": "railway_policy_news",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Media",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Economic and policy analysis of Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.reddit.com_r_IndianRailways_pNone_s1_c1",
+    "document_path": "https://www.reddit.com/r/IndianRailways",
+    "doc_category": "live_source",
+    "rule_type": "community_discussion",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Community discussions, photos and experiences"
+  },
+  {
+    "chunk_id": "https:__www.quora.com_topic_Indian-Railways_pNone_s1_c1",
+    "document_path": "https://www.quora.com/topic/Indian-Railways",
+    "doc_category": "live_source",
+    "rule_type": "community_qna",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Community",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Community Q&A related to Indian Railways"
+  },
+  {
+    "chunk_id": "https:__www.youtube.com_results?search_query=indian+railways_pNone_s1_c1",
+    "document_path": "https://www.youtube.com/results?search_query=indian+railways",
+    "doc_category": "live_source",
+    "rule_type": "video_reference",
+    "priority": 5,
+    "page_number": null,
+    "section_index": 1,
+    "authority": "Media / Community",
+    "is_static": false,
+    "effective_year": null,
+    "text": "Videos on train journeys, reviews, and railway operations"
+  }
+]

data/vector_store/rules_faiss.index

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2711bc634f94564782bb02777e5d8a3c7aad51aaa29d402edaad5437377b271c
+size 8908845

data/vector_store/rules_metadata.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:280a2346f507097a20afd84021e5687165155d57cc52fb204d8938b7dbb42a3f
+size 10668067

data_pipeline/build_static_lookup.py

@@ -0,0 +1,71 @@
+import json
+from pathlib import Path
+
+INP_DIR = Path("data/raw_docs/train_station")
+OUT_DIR = Path("data/static_lookup")
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+train_lookup = {}
+station_lookup = {}
+
+STATION_EXPANSIONS = {
+    " JN": " JUNCTION"
+}
+TRAIN_EXPANSIONS = {
+    " SF ": " SUPERFAST ",
+    " EXP ": " EXPRESS ",
+    " E ": " EXPRESS ",
+    " SPL ": " SPECIAL "
+}
+
+def normalize(text: str) -> str:
+    return " ".join(text.upper().split())
+
+def expand_all(name: str, expansions: dict) -> str:
+    expanded = f" {name} "
+    changed = True
+    while changed:
+        changed = False
+        for short, full in expansions.items():
+            if short in expanded:
+                expanded = expanded.replace(short, full)
+                changed = True
+
+    return normalize(expanded)
+
+def split_station(station_str):
+    if not station_str or "-" not in station_str:
+        return None, None
+    name, code = station_str.rsplit("-", 1)
+    return normalize(name), code.strip().upper()
+
+for file in INP_DIR.glob("*.json"):
+    with open(file, "r", encoding="utf-8") as f:
+        records = json.load(f)
+    for r in records:
+        tn = r.get("trainNumber")
+        tname = r.get("trainName")
+        if tn and tname:
+            original = normalize(tname)
+            expanded = expand_all(original, TRAIN_EXPANSIONS)
+            train_lookup[original.lower()] = tn
+            if expanded != original:
+                train_lookup[expanded.lower()] = tn
+        for stop in r.get("trainRoute", []):
+            raw = stop.get("stationName")
+            name, code = split_station(raw)
+            if name and code:
+                expanded = expand_all(name, STATION_EXPANSIONS)
+                station_lookup[name.lower()] = code
+                if expanded != name:
+                    station_lookup[expanded.lower()] = code
+
+with open(OUT_DIR / "trains_lookup.json", "w", encoding="utf-8") as f:
+    json.dump(train_lookup, f, indent=2, ensure_ascii=False)
+
+with open(OUT_DIR / "stations_lookup.json", "w", encoding="utf-8") as f:
+    json.dump(station_lookup, f, indent=2, ensure_ascii=False)
+
+print("✅ Static lookup files created")
+print("Trains:", len(train_lookup))
+print("Stations:", len(station_lookup))

data_pipeline/build_vector_store.py

@@ -0,0 +1,83 @@
+#Creating embeddings, vector index & implementing FAISS(manages vector indexes)
+
+import json
+from pathlib import Path
+import faiss
+from sentence_transformers import SentenceTransformer
+
+DATA_DIR = Path("data")
+CHUNKS_DIR = DATA_DIR / "chunks"
+VECTOR_DIR = DATA_DIR / "vector_store"
+VECTOR_DIR.mkdir(parents=True, exist_ok=True)
+CHUNK_FILES=[
+    CHUNKS_DIR/"core_docs_chunks.json",
+    CHUNKS_DIR/"circulars_chunks.json",
+    CHUNKS_DIR/"live_sources_chunks.json",
+]
+RULES_FAISS_INDEX_FILE = VECTOR_DIR / "rules_faiss.index"
+RULES_METADATA_FILE = VECTOR_DIR / "rules_metadata.json"
+LIVE_FAISS_INDEX_FILE = VECTOR_DIR / "live_faiss.index"
+LIVE_METADATA_FILE = VECTOR_DIR / "live_metadata.json"
+
+model=SentenceTransformer("all-MiniLM-L6-v2")
+rules_texts=[]
+rules_metadata=[]
+live_texts=[]
+live_metadata=[]
+for file in CHUNK_FILES:
+    if not file.exists():
+        continue
+    with open(file,"r",encoding="utf-8") as f:
+        records=json.load(f)
+    for rec in records:
+        meta = {
+            "chunk_id": rec["chunk_id"],
+            "document_path": rec["document_path"],
+            "doc_category": rec["doc_category"],
+            "rule_type": rec["rule_type"],
+            "priority": rec["priority"],
+            "page_number": rec["page_number"],
+            "section_index": rec["section_index"],
+            "authority": rec.get("authority"),
+            "is_static": rec.get("is_static"),
+            "effective_year": rec.get("effective_year"),
+            "text": rec.get("text")
+        }
+        if rec["doc_category"] == "live_source":
+            live_texts.append(rec["text"])
+            live_metadata.append(meta)
+        else:
+            rules_texts.append(rec["text"])
+            rules_metadata.append(meta)
+
+rules_embeddings = model.encode(
+    rules_texts,
+    batch_size=32,
+    show_progress_bar=True,
+    convert_to_numpy=True,
+    normalize_embeddings=True
+)
+rules_index = faiss.IndexFlatIP(rules_embeddings.shape[1])
+rules_index.add(rules_embeddings)
+faiss.write_index(rules_index,str(RULES_FAISS_INDEX_FILE))
+with open(RULES_METADATA_FILE, "w", encoding="utf-8") as f:
+    json.dump(rules_metadata, f, indent=2, ensure_ascii=False)
+
+live_embeddings = model.encode(
+    live_texts,
+    batch_size=32,
+    show_progress_bar=True,
+    convert_to_numpy=True,
+    normalize_embeddings=True
+)
+live_index = faiss.IndexFlatIP(live_embeddings.shape[1])
+live_index.add(live_embeddings)
+faiss.write_index(live_index, str(LIVE_FAISS_INDEX_FILE))
+with open(LIVE_METADATA_FILE,"w",encoding="utf-8") as f:
+    json.dump(live_metadata, f, indent=2, ensure_ascii=False)
+
+print(f"Rules chunks loaded     : {len(rules_texts)}")
+print(f"Live source chunks loaded: {len(live_texts)}")
+print(f"Rules FAISS index size  : {rules_index.ntotal}")
+print(f"Live FAISS index size   : {live_index.ntotal}")
+print("✅ Embeddings and FAISS index saved successfully.")

data_pipeline/chunking.py

@@ -0,0 +1,86 @@
+import json
+import re
+from pathlib import Path
+
+data_dir = Path("data")
+extract_dir = data_dir / "extracted_text"
+chunks_dir = data_dir / "chunks"
+chunks_dir.mkdir(parents=True, exist_ok=True)
+
+input_files = {
+    "core_docs": extract_dir / "core_docs.json",
+    "circulars": extract_dir / "circulars.json",
+    "live_sources": extract_dir / "live_sources.json",
+}
+
+CHUNK_WORDS = 600
+OVERLAP_WORDS = 120
+SECTION_PATTERN = re.compile(r"(?:^|\n)\s*(\d+\.\d+)\s+")
+
+def split_into_sections(text):
+    matches = list(SECTION_PATTERN.finditer(text))
+    if not matches:
+        return [text]
+    sections = []
+    for i, match in enumerate(matches):
+        start = match.start()
+        end = matches[i + 1].start() if i + 1 < len(matches) else len(text)
+        section_text = text[start:end].strip()
+        if section_text:
+            sections.append(section_text)
+    return sections
+
+def chunk_by_words(text, chunk_size, overlap):
+    words = text.split()
+    if len(words) <= chunk_size:
+        return [text]
+    chunks = []
+    step = chunk_size - overlap
+    for start in range(0, len(words), step):
+        end = start + chunk_size
+        chunk_words = words[start:end]
+        if chunk_words:
+            chunks.append(" ".join(chunk_words))
+    return chunks
+
+#Main splitting
+for source_name, input_file in input_files.items():
+    with open(input_file, "r", encoding="utf-8") as f:
+        records = json.load(f)
+    chunked_records = []
+    for rec in records:
+        text = rec.get("text", "").strip()
+        if not text:
+            continue
+        sections = split_into_sections(text)
+        for sec_idx, section in enumerate(sections, start=1):
+            section_chunks = chunk_by_words(
+                section,
+                CHUNK_WORDS,
+                OVERLAP_WORDS
+            )
+            for chunk_idx, chunk in enumerate(section_chunks, start=1):
+                chunk_record = {
+                    "chunk_id": (
+                        f"{rec['document_path'].replace('/', '_')}"
+                        f"_p{rec.get('page_number')}"
+                        f"_s{sec_idx}"
+                        f"_c{chunk_idx}"
+                    ),
+                    "document_path": rec.get("document_path"),
+                    "doc_category": rec.get("doc_category"),
+                    "rule_type": rec.get("rule_type"),
+                    "priority": rec.get("priority"),
+                    "authority": rec.get("authority"),
+                    "is_static": rec.get("is_static"),
+                    "effective_year": rec.get("effective_year"),
+                    "page_number": rec.get("page_number"),
+                    "section_index": sec_idx,
+                    "chunk_index": chunk_idx,
+                    "text": chunk
+                }
+                chunked_records.append(chunk_record)
+    output_file = chunks_dir / f"{source_name}_chunks.json"
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(chunked_records, f, indent=2, ensure_ascii=False)
+    print(f"✅ {source_name}: {len(chunked_records)} chunks created")

data_pipeline/extract_text.py

@@ -0,0 +1,142 @@
+import json
+import re
+from pathlib import Path
+from pypdf import PdfReader
+import pytesseract
+pytesseract.pytesseract.tesseract_cmd=r"D:\OCR\tesseract.exe"  # Give correct path
+import fitz
+from PIL import Image
+import io
+
+base_dir=Path(__file__).parent.parent
+# print(base_dir)
+data=base_dir/"data"
+raw_docs=data/"raw_docs"
+circulars=raw_docs/"circulars"
+core_docs=raw_docs/"core_docs"
+live_sources_file=raw_docs/"live_sources"/"live_sources.json"
+metadata_file=raw_docs/"meta_data.json"
+
+output_dir=data/"extracted_text"
+output_dir.mkdir(parents=True,exist_ok=True)
+
+with open(metadata_file,"r",encoding="utf-8") as f:            # Reads core and circular folders metadata(custom made not program generated)
+    file_metadata=json.load(f)
+# print(file_metadata["core_docs/Railways_Act_1989.pdf"])
+
+def clean_text(text):    #Converts to cleaned str
+    text = re.sub(r"\n{3,}","\n\n",text)
+    text=re.sub(r"[ ]{2,} | \t+"," ",text)
+    return text.strip()
+
+def ocr_pdf(pdf_path):   #Reads images in pdfs
+    doc = fitz.open(pdf_path)
+    pages_text = []
+    for i, page in enumerate(doc, start=1):
+        pix = page.get_pixmap(dpi=300)
+        img = Image.open(io.BytesIO(pix.tobytes("png")))
+        text = pytesseract.image_to_string(img, lang="eng")
+        if text.strip():
+            pages_text.append((i, text))
+    return pages_text
+
+def extract_pdf(pdf_path, relative_key):
+    records = []
+    meta = file_metadata.get(relative_key)
+    if not meta:
+        print(f"[WARNING] Metadata missing for {relative_key}")
+        return records
+    reader = PdfReader(str(pdf_path))
+    has_text = False
+    for page in reader.pages:
+        text = page.extract_text()
+        if text and text.strip():
+            has_text = True
+            break
+    if has_text:
+        for page_no, page in enumerate(reader.pages, start=1):
+            text = page.extract_text()
+            if not text:
+                continue
+            records.append({
+                "document_id": relative_key.replace("/", "__"),
+                "document_name": pdf_path.name,
+                "document_path": relative_key,
+                "doc_category": meta.get("doc_category"),
+                "rule_type": meta.get("rule_type"),
+                "priority": meta.get("priority"),
+                "authority": meta.get("authority"),
+                "description": meta.get("description"),
+                "is_static": meta.get("is_static"),
+                "effective_year": meta.get("effective_year"),
+                "page_number": page_no,
+                "text": clean_text(text),
+                "extraction_method": "text"
+            })
+    else:
+        ocr_pages = ocr_pdf(pdf_path)
+        if not ocr_pages:
+            print(f"[OCR FAILED] {relative_key}")
+        for page_no, ocr_text in ocr_pages:
+            records.append({
+                "document_id": relative_key.replace("/", "__"),
+                "document_name": pdf_path.name,
+                "document_path": relative_key,
+                "doc_category": meta.get("doc_category"),
+                "rule_type": meta.get("rule_type"),
+                "priority": meta.get("priority"),
+                "authority": meta.get("authority"),
+                "description": meta.get("description"),
+                "is_static": meta.get("is_static"),
+                "effective_year": meta.get("effective_year"),
+                "page_number": page_no,
+                "text": clean_text(ocr_text),
+                "extraction_method": "ocr"
+            })
+    return records
+
+
+def process_folder(base_folder,category_name):
+    all_records=[]
+    for pdf_file in base_folder.rglob("*.pdf"):
+        relative_path=pdf_file.relative_to(base_folder).as_posix()
+        relative_key=f"{category_name}/{relative_path}"
+        print(f"Processing: {relative_key}")
+        records=extract_pdf(pdf_file, relative_key)    #Calls extract function
+        if not records:
+            print(f"[SKIPPED] No extractable text: {relative_key}")
+        all_records.extend(records)
+    return all_records
+
+def process_live_sources():
+    with open(live_sources_file,"r",encoding="utf-8") as f:
+        live_sources=json.load(f)
+    records=[]
+    for item in live_sources:
+        record={
+            "document_name": item.get("name"),
+            "document_path": item.get("url"),
+            "doc_category": "live_source",
+            "rule_type": item.get("rule_type"),
+            "priority": item.get("priority"),
+            "authority": item.get("authority"),
+            "description": item.get("description"),
+            "is_static": False,
+            "effective_year": None,
+            "page_number": None,
+            "text":clean_text(item.get("description",""))
+        }
+        records.append(record)
+    return records
+
+if __name__=="__main__":
+    core_records=process_folder(core_docs,"core_docs")
+    circular_records=process_folder(circulars,"circulars")
+    live_records=process_live_sources()
+    with open(output_dir/"core_docs.json","w",encoding="utf-8") as f:
+        json.dump(core_records,f,indent=2,ensure_ascii=False)
+    with open(output_dir/"circulars.json","w",encoding="utf-8") as f:
+        json.dump(circular_records,f,indent=2,ensure_ascii=False)
+    with open(output_dir/"live_sources.json","w",encoding="utf-8") as f:
+        json.dump(live_records,f,indent=2,ensure_ascii=False)
+    print("\n✅ Document ingestion completed successfully.")

helpers/live_sources.py

@@ -0,0 +1,42 @@
+# Retrieves links for all modules
+import json
+from pathlib import Path
+import faiss
+from sentence_transformers import SentenceTransformer
+DATA_DIR = Path("data")
+VECTOR_DIR = DATA_DIR / "vector_store"
+LIVE_FAISS_INDEX_PATH = VECTOR_DIR / "live_faiss.index"
+LIVE_METADATA_PATH = VECTOR_DIR / "live_metadata.json"
+index = faiss.read_index(str(LIVE_FAISS_INDEX_PATH))
+with open(LIVE_METADATA_PATH, "r", encoding="utf-8") as f:
+    METADATA = json.load(f)
+model = SentenceTransformer("all-MiniLM-L6-v2")
+def retrieve_live_sources(
+    query: str,
+    *,
+    top_k: int = 2,
+    search_k: int = 2000
+):
+    query_embedding = model.encode(
+        [query],
+        normalize_embeddings=True,
+        convert_to_numpy=True
+    )
+    scores, indices = index.search(query_embedding, search_k)
+    results = []
+    seen_urls = set()
+    for score, idx in zip(scores[0], indices[0]):
+        meta = METADATA[idx]
+        url = meta.get("document_path")
+        if not url or url in seen_urls:
+            continue
+        seen_urls.add(url)
+        results.append({
+            "url": url,
+            "authority": meta.get("authority"),
+            "description": meta.get("text"),
+            "similarity": float(score)
+        })
+        if len(results) >= top_k:
+            break
+    return results

modules/general_chat.py

@@ -0,0 +1,62 @@
+# General module
+import os
+from openai import OpenAI
+
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+if not client:
+    raise RuntimeError("OPENAI_API_KEY is not set")
+MODEL_NAME = "gpt-4.1-mini"
+GENERAL_SYSTEM_PROMPT = (
+    "You are a knowledgeable, helpful AI assistant.\n"
+    "You can answer general questions, explain concepts clearly, "
+    "and respond politely in a conversational manner.\n"
+    "If a question is ambiguous, ask for clarification.\n"
+    "Do not claim access to private, real-time, or restricted systems."
+)
+def get_relevance(route, module_name):
+    for m in route.get("module_preferences", []):
+        if m["module"] == module_name:
+            return m["relevance"]
+    return 0.0
+def answer_general_query(query,route,mode="module"):
+    if mode == "failsafe":
+        response = client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[
+                {"role": "system", "content": GENERAL_SYSTEM_PROMPT},
+                {"role": "user", "content": query}],temperature=0.6)
+        return {
+            "answer": response.choices[0].message.content.strip(),
+            "has_answer": True,
+            "meta": {"mode": "failsafe"}}
+    gen_rel = get_relevance(route, "general")
+    rag_rel = get_relevance(route, "railway_rag")
+    api_rel = get_relevance(route, "live_data_apis")
+    MIN_GENERAL_RELEVANCE = 0.30
+    DOMINANCE_MARGIN = 0.10
+    if gen_rel < MIN_GENERAL_RELEVANCE:                        #Check 1
+        return {
+            "answer": None,
+            "has_answer": False,
+            "meta": {}
+        }
+    if gen_rel < max(rag_rel, api_rel) + DOMINANCE_MARGIN:      #Check 2
+        return {
+            "answer": None,
+            "has_answer": False,
+            "meta": {}
+        }
+    response = client.chat.completions.create(
+        model=MODEL_NAME,
+        messages=[
+            {"role": "system", "content": GENERAL_SYSTEM_PROMPT},
+            {"role": "user", "content": query}
+        ],
+        temperature=0.6
+    )
+    answer_text = response.choices[0].message.content.strip()
+    return {
+        "answer": answer_text,
+        "has_answer": True,
+        "meta": {}
+    }

modules/link_answer.py

@@ -0,0 +1,16 @@
+# Link only module
+from helpers.live_sources import retrieve_live_sources
+from helpers.live_sources import retrieve_live_sources
+def run(query, num_of_links):
+    sources = retrieve_live_sources(query, top_k=num_of_links)
+    if not sources:
+        return []
+    return [
+        {
+            "url": s["url"],
+            "authority": s.get("authority"),
+            "similarity": s.get("similarity")
+        }
+        for s in sources
+        if s.get("url")
+    ]

modules/live_data_apis.py

@@ -0,0 +1,396 @@
+#APIs
+import os
+import json
+import requests
+from openai import OpenAI
+from dotenv import load_dotenv
+load_dotenv()
+from datetime import datetime, timezone, timedelta
+
+with open("data/static_lookup/stations_lookup.json", "r", encoding="utf-8") as f:
+    STATION_LOOKUP = json.load(f)
+with open("data/static_lookup/trains_lookup.json", "r", encoding="utf-8") as f:
+    TRAIN_LOOKUP = json.load(f)
+RAPIDAPI_KEY = os.getenv("RAPIDAPI_KEY")
+if not RAPIDAPI_KEY:
+    raise RuntimeError("RAPIDAPI_KEY is not set")
+RAPIDAPI_HOST = "irctc1.p.rapidapi.com"
+BASE_URL = "https://irctc1.p.rapidapi.com"
+HEADERS = {
+    "x-rapidapi-key": RAPIDAPI_KEY,
+    "x-rapidapi-host": RAPIDAPI_HOST
+}
+llm = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+LLM_MODEL = "gpt-4.1-mini"
+ENTITY_PROMPT = """
+You extract structured railway-related information from user queries.
+
+Return ONLY valid JSON.
+Do NOT infer or assume missing information.
+Do NOT guess defaults.
+Extract a field ONLY if it is explicitly mentioned by the user.
+If a value is not present, use null or an empty list as specified.
+
+Supported intents (choose ONE):
+- train_live_status              (live running status of a train)
+- train_schedule                 (timetable / schedule of a train)
+- trains_between_stations        (trains running between two stations)
+- seat_availability              (seat/berth availability enquiry)
+- fare_enquiry                   (ticket fare enquiry)
+- pnr_status                     (PNR booking status)
+- live_station                   (live arrivals/departures at a station)
+- trains_by_station              (list of trains passing a station)
+- search_train                   (search trains by name or number)
+- search_station                 (search stations by name)
+- unknown
+
+JSON schema:
+{
+  "intent": string,
+
+  "train_numbers": [string],        // 5-digit train numbers only
+  "pnr_numbers": [string],          // 10-digit PNR numbers only
+
+  "stations": [string],             // station names or codes exactly as mentioned
+
+  "journey": {
+    "from": string | null,          // source station name/code if explicitly mentioned
+    "to": string | null             // destination station name/code if explicitly mentioned
+  },
+
+  "date": string | null,            // travel date if mentioned (keep original format)
+  "class_type": string | null,      // class if mentioned (e.g., 2A, 3A, SL, CC)
+  "quota": string | null,           // quota if mentioned (e.g., GN, Tatkal)
+  "hours": integer | null           // time window in hours if explicitly mentioned
+}
+
+Important rules:
+- Do NOT convert station names to codes.
+- Do NOT normalize dates or class names.
+- Do NOT infer intent from missing data.
+- Do NOT fill defaults.
+- If multiple values exist, extract all where applicable.
+"""
+
+def extract_with_llm(query):    
+    resp = llm.chat.completions.create(
+        model=LLM_MODEL,
+        temperature=0,
+        messages=[
+            {"role": "system", "content": ENTITY_PROMPT},
+            {"role": "user", "content": query}
+        ]
+    )
+    return json.loads(resp.choices[0].message.content)
+
+def call_api(path: str, params):
+    print("\n[API CALL]")
+    print("URL:", BASE_URL + path)
+    print("PARAMS:", params)
+    try:
+        r = requests.get(
+            BASE_URL + path,
+            headers=HEADERS,
+            params=params,
+            timeout=10
+        )
+        print("STATUS:", r.status_code)
+        print("RESPONSE:", r.text[:300])
+        if r.status_code != 200:
+            return None, r.headers
+        return r.json().get("data"), r.headers
+    except Exception as e:
+        print("❌ API EXCEPTION:", e)
+        return None, None
+
+def resolve_train_number(value: str):
+    if not value:
+        return None
+    clean = value.lower().strip()
+    if clean.isdigit() and len(clean) == 5:
+        return clean
+    return TRAIN_LOOKUP.get(clean)
+def resolve_station_code_local(name: str):
+    if not name:
+        return None
+    clean = name.lower().strip()
+    if clean in STATION_LOOKUP:
+        return STATION_LOOKUP[clean]
+    return resolve_station_code(name)
+
+def resolve_station_code(name: str):
+    if not name:
+        return None
+    clean = name.lower().replace(" station", "").strip()
+    data, _ = call_api("/api/v1/searchStation", {"query": clean})
+    if not data:
+        return None
+    for s in data:
+        station_name = s.get("name", "").lower()
+        if station_name == clean:
+            return s.get("code")
+    for s in data:
+        station_name = s.get("name", "").lower()
+        if station_name.startswith(clean):
+            return s.get("code")
+    return data[0].get("code")
+
+# Determine freshness (for additional links)
+def determine_freshness(headers):
+    if not headers:
+        return "unknown"    
+    date_header = headers.get("Date")
+    if not date_header:
+        return "unknown"
+    try:
+        response_time = datetime.strptime(
+            date_header, "%a, %d %b %Y %H:%M:%S GMT"
+        ).replace(tzinfo=timezone.utc)
+    except Exception:
+        return "unknown"
+    if datetime.now(timezone.utc) - response_time > timedelta(minutes=60):
+        return "stale"
+    return "fresh"
+
+# Date format correction
+
+def normalize_date_for_api(date_str):
+    for fmt in ("%d-%m-%Y", "%Y-%m-%d", "%d/%m/%Y"):
+        try:
+            return datetime.strptime(date_str,fmt).strftime("%d-%m-%Y")
+        except ValueError:
+            continue
+    return None
+
+#APIs
+
+def get_train_live_status(e):
+    return call_api("/api/v1/liveTrainStatus", {"trainNo": e["train_number"]})
+def get_train_schedule(e):
+    return call_api("/api/v1/getTrainSchedule", {"trainNo": e["train_number"]})
+def get_trains_between_stations(e):
+    return call_api("/api/v3/trainBetweenStations", {
+        "fromStationCode": e["from_station"],
+        "toStationCode": e["to_station"],
+        "dateOfJourney": e["date"]
+    })
+def get_pnr_status(e):
+    return call_api("/api/v3/getPNRStatus", {"pnrNumber": e["pnr"]})
+def get_live_station(e):
+    return call_api("/api/v3/getLiveStation", {
+        "fromStationCode": e["station_code"],
+        "hours": e.get("hours", 2)
+    })
+def get_trains_by_station(e):
+    return call_api("/api/v3/getTrainsByStation", {"stationCode": e["station_code"]})
+def search_train(e):
+    return call_api("/api/v1/searchTrain", {"query": e["query"]})
+def search_station(e):
+    return call_api("/api/v1/searchStation", {"query": e["query"]})
+def get_seat_availability(e):
+    return call_api("/api/v1/checkSeatAvailability", {
+        "trainNo": e["train_number"],
+        "fromStationCode": e["from_station"],
+        "toStationCode": e["to_station"],
+        "date": e["date"],
+        "classType": e["class_type"],
+        "quota": e.get("quota", "GN")
+    })
+def get_seat_availability_v2(e):
+    return call_api("/api/v2/checkSeatAvailability", {
+        "trainNo": e["train_number"],
+        "fromStationCode": e["from_station"],
+        "toStationCode": e["to_station"],
+        "date": e["date"],
+        "classType": e["class_type"],
+        "quota": e.get("quota", "GN")
+    })
+def get_fare(e):
+    return call_api("/api/v1/getFare", {
+        "trainNo": e["train_number"],
+        "fromStationCode": e["from_station"],
+        "toStationCode": e["to_station"],
+        "date": e["date"],
+        "classType": e["class_type"],
+        "quota": e.get("quota", "GN")
+    })
+
+#Check whether apis match
+
+
+# Calling crct API
+# Mapping intent to API
+
+INTENT_TO_API = {
+    "train_live_status": {
+        "required": ["train_number"],
+        "handler": get_train_live_status,
+        "fallback": "train_schedule"
+    },
+    "train_schedule": {
+        "required": ["train_number"],
+        "handler": get_train_schedule,
+        "fallback": None
+    },
+    "trains_between_stations": {
+        "required": ["from_station", "to_station", "date"],
+        "handler": get_trains_between_stations,
+        "fallback": None
+    },
+    "seat_availability": {
+        "required": ["train_number", "from_station", "to_station", "date", "class_type"],
+        "handler": get_seat_availability,
+        "fallback": "seat_availability_v2"
+    },
+    "seat_availability_v2": {
+        "required": ["train_number", "from_station", "to_station", "date", "class_type"],
+        "handler": get_seat_availability_v2,
+        "fallback": None
+    },
+    "fare_enquiry": {
+        "required": ["train_number", "from_station", "to_station", "date", "class_type"],
+        "handler": get_fare,
+        "fallback": None
+    },
+    "pnr_status": {
+        "required": ["pnr"],
+        "handler": get_pnr_status,
+        "fallback": None
+    },
+    "live_station": {
+        "required": ["station_code"],
+        "handler": get_live_station,
+        "fallback": "trains_by_station"
+    },
+    "trains_by_station": {
+        "required": ["station_code"],
+        "handler": get_trains_by_station,
+        "fallback": None
+    },
+    "search_train": {
+        "required": ["query"],
+        "handler": search_train,
+        "fallback": None
+    },
+    "search_station": {
+        "required": ["query"],
+        "handler": search_station,
+        "fallback": None
+    }
+}
+
+# Main
+
+def answer_with_live_data(query):
+    parsed = extract_with_llm(query)
+    intent = parsed.get("intent", "unknown")
+    if intent not in INTENT_TO_API:
+        return {"answer": None, "has_answer": False, "meta": {"status":"nothing"}}
+    entity = {"query": query}
+    if parsed.get("train_numbers"):
+        tn = resolve_train_number(parsed["train_numbers"][0])
+        if tn:
+            entity["train_number"] = tn
+    if parsed.get("pnr_numbers"):
+        entity["pnr"] = parsed["pnr_numbers"][0]
+    resolved = []
+    for s in parsed.get("stations", []):
+        code = resolve_station_code_local(s)
+        if code:
+            resolved.append(code)
+    if len(resolved) == 1:
+        entity["station_code"] = resolved[0]
+    journey = parsed.get("journey")
+    if journey and journey.get("from") and journey.get("to"):
+        f = resolve_station_code_local(journey["from"])
+        t = resolve_station_code_local(journey["to"])
+        if f and t:
+            entity["from_station"] = f
+            entity["to_station"] = t
+    if parsed.get("date"):
+        normalized = normalize_date_for_api(parsed["date"])
+        if normalized:
+            entity["date"] = normalized
+    elif intent == "trains_between_stations":
+        entity["date"] = datetime.now(timezone.utc).strftime("%d-%m-%Y")
+    for k in ["class_type", "quota", "hours"]:
+        if parsed.get(k) is not None:
+            entity[k] = parsed[k]
+    api = INTENT_TO_API[intent]
+    # required, missing
+    required = api["required"]
+    missing = [k for k in required if k not in entity]
+    if missing:
+        return {
+            "answer": None,
+            "has_answer": False,
+            "meta": {
+                "status": "need_input",
+                "reason": "missing_required_fields",
+                "intent": intent,
+                "missing_fields": missing,
+                "partial_entity": entity
+            }
+        }
+    # Primary API call
+    data, headers= api["handler"](entity)
+    fallback_used = False
+    # Fallback handling
+    if not data and api.get("fallback"):
+        fb = INTENT_TO_API[api["fallback"]]
+        if all(k in entity for k in fb["required"]):
+            data, headers= fb["handler"](entity)
+            if data:
+                fallback_used = True
+    # Freshness detection (safe, signal only)
+    freshness = determine_freshness(headers)
+    if not data:
+        return {"answer": None, "has_answer": False, "meta": {"status": "api_failed","intent": intent}}
+    return {
+        "answer":data,
+        "has_answer": True,
+        "meta": {
+            "status": "ok",
+            "intent": intent,
+            "fallback_used": fallback_used,
+            "freshness": freshness,
+            "resolved": {
+                "train_number": entity.get("train_number"),
+                "station_code": entity.get("station_code"),
+                "from_station": entity.get("from_station"),
+                "to_station": entity.get("to_station"),
+                "date": entity.get("date")
+            }
+        }
+    }
+#TEST
+# if __name__ == "__main__":
+#     print("=" * 80)
+#     print("API TEST STARTED")
+#     print("RAPIDAPI_KEY exists:", bool(RAPIDAPI_KEY))
+#     print("RAPIDAPI_HOST:", RAPIDAPI_HOST)
+#     print("=" * 80)
+
+#     test_queries = [
+#          # search station (sanity check)
+#         "Kozhikode trains",
+#         # search train (sanity check)
+    
+#     ]
+
+#     for q in test_queries:
+#         print("\n" + "-" * 60)
+#         print("Query:", q)
+
+#         result = answer_with_live_data(q)
+
+#         print("HAS ANSWER:", result.get("has_answer"))
+#         print("META:", result.get("meta"))
+
+#         if result.get("has_answer"):
+#             print("DATA PREVIEW:")
+#             print(result["answer"][:500])  # prevent huge spam
+#         else:
+#             print("❌ NO DATA RETURNED")
+
+#     print("\nAPI TEST COMPLETED")

modules/railway_rag/railway_base_rag.py

@@ -0,0 +1,112 @@
+# Railway Rules RAG Module
+import os
+from openai import OpenAI
+from dotenv import load_dotenv
+load_dotenv()
+from modules.railway_rag.retrieval_engine import retrieve_rules
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+if not client:
+    raise RuntimeError("OPENAI_API_KEY is not set")
+MODEL_NAME = "gpt-4.1-mini"
+SYSTEM_PROMPT = (
+    "You are a railway rules assistant.\n"
+    "You must answer the user question ONLY if the provided railway rules, clearly and directly contain the answer.\n"
+    "Do not use any information outside the given rules.\n"
+    "Do not speculate or infer missing details.\n"
+    "If the rules do not clearly answer the question, do not attempt to answer."
+)
+def build_context(chunks):                  #Make all chunks into one block(in dec order of similarity for chunks)
+    blocks = []                             #Creates text out of all retrieved chunks, For llm to read it like a document
+    for i, c in enumerate(chunks, start=1):
+        blocks.append(
+            f"[Source {i}]\n"
+            f"Document: {c['document_path']}\n"
+            f"Authority: {c.get('authority')}\n"
+            f"Page: {c.get('page_number')}, "
+            f"Section: {c.get('section_index')}\n"
+            f"Text:\n{c['text']}\n"
+        )
+    return "\n\n".join(blocks)
+def extract_citations(chunks):               #Citations
+    return [
+        {
+            "document_path": c.get("document_path"),
+            "authority": c.get("authority"),
+            "rule_type": c.get("rule_type"),
+            "page_number": c.get("page_number"),
+            "section_index": c.get("section_index"),
+            "effective_year":c.get("effective_year")
+        }
+        for c in chunks            #For each chunk
+    ]
+
+def estimate_confidence(chunks):    #Find Confidence
+    if not chunks:
+        return 0.0
+    # Collect similarities (LLM used all of them)
+    sims = [c.get("similarity", 0.0) for c in chunks]
+    # Best supporting rule
+    top_sim = max(sims)
+    # No strong rule at all → low confidence
+    if top_sim < 0.45:
+        return round(top_sim * 0.8, 2)
+    # Consider only chunks that genuinely support the answer
+    # (close enough to the best one)
+    support_sims = [
+        s for s in sims
+        if s >= top_sim - 0.10
+    ]
+    # Mean support from all relevant chunks
+    support_mean = sum(support_sims) / len(support_sims)
+    # Final confidence
+    confidence = (
+        0.75 * top_sim +
+        0.25 * support_mean
+    )
+    return round(min(confidence, 0.9), 2)
+
+def answer_with_rag(query: str):
+    chunks = retrieve_rules(query)
+    if not chunks:  # No chunks
+        return {
+            "answer": None,
+            "has_answer": False,
+            "meta": {"confidence": 0.0,}
+        }
+    context=build_context(chunks)
+    user_prompt=(
+        f"User question:\n{query}\n\n"
+        f"Relevant railway rules:\n{context}\n\n"
+        "Instructions:\n"
+        "- Answer strictly using the rules above.\n"
+        "- Do not add external knowledge.\n"
+    )
+    response = client.chat.completions.create(
+        model=MODEL_NAME,
+        messages=[
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": user_prompt}
+        ],
+        temperature=0.2
+    )
+    answer_text = response.choices[0].message.content.strip()
+    confidence=estimate_confidence(chunks)
+    if not answer_text:
+        return {
+            "answer": None,
+            "has_answer": False,
+            "meta": {
+                "confidence": confidence
+            }
+        }
+    return {
+        "answer": answer_text,
+        "has_answer": confidence>=0.45,
+        "meta": {
+            "confidence": confidence,
+            "citations": extract_citations(chunks),
+            "rule_types": list({c.get("rule_type") for c in chunks if c.get("rule_type")})
+        }
+    }
+if __name__ == "__main__":
+    print("✅ Railway base RAG module ready.")

modules/railway_rag/retrieval_engine.py

@@ -0,0 +1,99 @@
+#Helps railwaay_base_rag 
+import json
+from pathlib import Path
+import faiss
+from sentence_transformers import SentenceTransformer
+
+TOP_K_RETRIEVE = 50
+TOP_K_FINAL = 10
+SIMILARITY_THRESHOLD = 0.30
+PRIORITY_WEIGHT = 0.15
+RULE_MATCH_WEIGHT = 0.10
+RECENCY_WEIGHT = 0.05
+QUESTION_TYPE_WEIGHT = 0.10   
+DATA_DIR = Path("data")
+VECTOR_DIR = DATA_DIR / "vector_store"
+RULES_FAISS_INDEX_PATH = VECTOR_DIR / "rules_faiss.index"
+RULES_METADATA_PATH = VECTOR_DIR / "rules_metadata.json"
+index = faiss.read_index(str(RULES_FAISS_INDEX_PATH))
+with open(RULES_METADATA_PATH, "r", encoding="utf-8") as f:
+    METADATA = json.load(f)
+model = SentenceTransformer("all-MiniLM-L6-v2")
+assert index.ntotal == len(METADATA), "Index and metadata size mismatch"
+
+#Scoring helpers
+def rule_match_score(query: str, rule_type: str) -> float:
+    if not rule_type:
+        return 0.0
+    return 1.0 if rule_type.lower() in query.lower() else 0.0
+def recency_score(year):
+    if year is None:
+        return 0.0
+    return min((year - 2000) / 25, 1.0)
+def detect_question_type(query: str) -> str:
+    q = query.lower()
+    if any(x in q for x in ["what is", "define", "meaning of"]):
+        return "definition"
+    if any(x in q for x in ["can i", "allowed", "permitted"]):
+        return "permission"
+    if any(x in q for x in ["not allowed", "prohibited", "shall not"]):
+        return "prohibition"
+    if any(x in q for x in ["how to", "procedure", "process"]):
+        return "procedure"
+    if any(x in q for x in ["penalty", "fine", "punishment", "liable"]):
+        return "penalty"
+    return "general"
+def question_type_bonus(qtype: str, text: str) -> float:
+    t = text.lower()
+    if qtype == "definition" and any(x in t for x in ["means", "defined as"]):
+        return QUESTION_TYPE_WEIGHT
+    if qtype == "permission" and any(x in t for x in ["may", "permitted", "allowed"]):
+        return QUESTION_TYPE_WEIGHT
+    if qtype == "prohibition" and any(x in t for x in ["shall not", "prohibited"]):
+        return QUESTION_TYPE_WEIGHT
+    if qtype == "procedure" and any(x in t for x in ["procedure", "steps", "shall be"]):
+        return QUESTION_TYPE_WEIGHT
+    if qtype == "penalty" and any(x in t for x in ["penalty", "fine", "liable"]):
+        return QUESTION_TYPE_WEIGHT
+    return 0.0
+
+
+def retrieve_rules(query: str):
+    query_embedding = model.encode(
+        [query],
+        normalize_embeddings=True,
+        convert_to_numpy=True
+    )
+    scores, indices = index.search(query_embedding, TOP_K_RETRIEVE)
+    qtype = detect_question_type(query)
+    candidates = []
+    for i, idx in enumerate(indices[0]):
+        similarity = float(scores[0][i])
+        if similarity < SIMILARITY_THRESHOLD:
+            continue
+        meta = METADATA[idx]
+        priority_bonus = (1 / meta["priority"]) * PRIORITY_WEIGHT
+        rule_bonus = rule_match_score(query, meta["rule_type"]) * RULE_MATCH_WEIGHT
+        recency_bonus = recency_score(meta.get("effective_year")) * RECENCY_WEIGHT
+        qtype_bonus = question_type_bonus(qtype, meta["text"])
+        final_score = (
+            similarity
+            + priority_bonus
+            + rule_bonus
+            + recency_bonus
+            + qtype_bonus
+        )
+        candidates.append({
+            "final_score": round(final_score, 4),
+            "similarity": round(similarity, 4),
+            "chunk_id": meta["chunk_id"],
+            "document_path": meta["document_path"],
+            "rule_type": meta["rule_type"],
+            "priority": meta["priority"],
+            "authority": meta["authority"],
+            "page_number": meta["page_number"],
+            "section_index": meta.get("section_index"),
+            "text": meta["text"]
+        })
+    candidates.sort(key=lambda x: x["final_score"], reverse=True)
+    return candidates[:TOP_K_FINAL]

requirements.txt

@@ -0,0 +1,20 @@
+# UI
+gradio
+
+# API backend (kept for completeness)
+fastapi
+pydantic
+
+# LLM & RAG
+openai
+faiss-cpu
+sentence-transformers
+
+# Data & utilities
+requests
+python-dotenv
+
+# Document processing (PDFs)
+pypdf
+PyMuPDF
+Pillow

runtime.txt

@@ -0,0 +1 @@
+python-3.10