added chatbot codes

README.md

@@ -1,3 +1,50 @@
+## Project Overview
+This respository was created as part for an internship report, containing the source codes and respurces in developing the Retrieval-Augmented Generation(RAG) banking chatbot.  
+
+RAG allows the chatbot to answer questions based on the knowledge base instead of only using the Large Language Model (LLM). 
+
+## Objectives
+- Develop a chatbot capable of answering questions based on public dataset(s)
+- Implement a Retrieval-Augmented Generation (RAG) pipeline
+- Generate embeddings for efficient document retrieval
+- Build a vector database
+- Improve response accuracy by providing relevant context to the language model
+- Implement advanced requirements
+- Evaluate the chatbot's performance 
+
+## Installation
+Virtual environment:  
+python -m venv venv  
+venv\Scripts\activate  
+
+Libraries:  
+pip install -r requirements.txt  
+
+Run the chatbot:  
+streamlit run app.py  
+
+## How it works
+The chatbot follows a Retrieval-Augmented Generation (RAG) workflow:
+
+1. Documents are collected and preprocessed.
+2. The documents are split into smaller text chunks.
+3. Each chunk is converted into vector embeddings.
+4. The embeddings are stored in a vector database.
+5. When a user submits a query:
+   - The query is embedded into a vector.
+   - The vector database retrieves the most relevant document chunks.
+   - The retrieved context is combined with the user's query.
+   - The LLM generates an appropriate response using the retrieved information.
+
+## Requirements
+- Python 3.10 or above
+- LangChain
+- Ollama
+- Vector database (ChromaDB)
+- Sentence Transformer / Embedding model
+- python-dotenv
+- Other dependencies listed in `requirements.txt`
+
 ---
 license: mit
 ---

agentic_workflow/config.py

@@ -0,0 +1,8 @@
+CHROMA_PATH = "./chroma_db"
+COLLECTION_NAME = "knowledge"
+CORRECTIONS_COLLECTION = "corrections"
+EMBED_MODEL = "multi-qa-mpnet-base-dot-v1"   
+LLM_MODEL = "llama3.2"           
+CONFIDENCE_THRESHOLD = 0.75
+MAX_RETRIES = 3
+TOP_K = 3

agentic_workflow/evaluator.py

@@ -0,0 +1,60 @@
+from agentic_workflow.config import LLM_MODEL
+from langchain_ollama import ChatOllama
+import re
+import json
+from langchain_core.messages import HumanMessage, SystemMessage
+
+llm = ChatOllama(model=LLM_MODEL, temperature=0)
+
+def evaluate_response(query: str, response: str) -> dict:
+    """
+    Scores the draft response and decides whether to retry.
+ 
+    Returns a dict with keys:
+      - score:       int 1-10
+      - pass:        bool  → True means use this response
+      - reason:      str   → short explanation
+      - suggestion:  str   → how to improve (if not passing)
+    """
+    system = """
+        You are a strict response quality evaluator. Given a user query and a draft response, return ONLY valid JSON:
+        - "score": integer 1-10 (10 = excellent)
+        - "pass": true if score >= 7, false otherwise
+        - "reason": one-sentence explanation
+        - "suggestion": if not passing, a short instruction to improve the response; else ""
+        
+        Return ONLY the JSON object. No explanation.
+        """
+ 
+    response_eval = llm.invoke([
+        SystemMessage(content=system),
+        HumanMessage(content=f"Query: {query}\n\nDraft response: {response}")
+    ])
+    raw = response_eval.content.strip()
+
+    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
+
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        parsed = {}
+
+    return {
+        "score": 5,
+        "pass": False,
+        "reason": "Evaluation failed",
+        "suggestion": "Retry with more detail",
+        **parsed
+    }
+
+# result = evaluate_response(
+#     query="What is the refund policy?",
+#     response="According to the context provided earlier, the cancellation procedure involves a notice period of 30 days. The premium paid by you will be returned on a pro-rata basis or 25% of the annual premium, whichever is higher, will be retained. Any cancellation request sent after 30 days of commencement of the policy will be refunded on a pro-rata basis."
+# )
+# print("Test 1 (good response):", result)
+
+# result = evaluate_response(
+#     query="What is the refund policy?",
+#     response="You can get a refund if you contact support."
+# )
+# print("Test 3 (vague response):", result)

agentic_workflow/generator.py

@@ -0,0 +1,38 @@
+import json
+import chromadb
+from agentic_workflow.config import LLM_MODEL
+from langchain_ollama import ChatOllama
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+
+llm = ChatOllama(model=LLM_MODEL, temperature=0)
+
+def generate_llm_response(query: str, context: list[str], history: list[dict], suggestion: str = "") -> str:
+    system_parts = ["You are a helpful assistant."]
+    if suggestion:
+        system_parts.append(f"Improvement hint from previous attempt: {suggestion}")
+
+    messages = [SystemMessage(content=" ".join(system_parts))]
+
+    # Convert history dicts to LangChain message objects
+    for turn in history:
+        if turn["role"] == "user":
+            messages.append(HumanMessage(content=turn["content"]))
+        elif turn["role"] == "assistant":
+            messages.append(AIMessage(content=turn["content"]))
+
+    # Append context to the user query if available
+    user_content = query
+    if context:
+        user_content = "Context:\n" + "\n".join(context) + "\n\nQuestion: " + query
+
+    messages.append(HumanMessage(content=user_content))
+
+    response = llm.invoke(messages)
+    return response.content
+
+# result = generate_llm_response(
+#     query="What is the refund policy?",
+#     context=[],
+#     history=[]
+# )
+# print("Test 2:", result)

agentic_workflow/intent.py

@@ -0,0 +1,55 @@
+import json
+import chromadb
+from agentic_workflow.config import LLM_MODEL
+from langchain_ollama import ChatOllama
+import re
+from langchain_core.messages import HumanMessage, SystemMessage
+
+client = chromadb.PersistentClient(path="./chroma_db")
+llm = ChatOllama(model=LLM_MODEL, temperature=0)
+
+# intent analysis
+def analyse_intent(query: str) -> dict:
+
+    system = """
+        You are an intent classifier. Given a user query, return ONLY valid JSON with:
+        - "intent": one of "factual", "conversational", "follow_up", "sensitive"
+        - "needs_context": true if retrieval from a knowledge base is needed, false otherwise
+        - "needs_history": true if this looks like a follow-up to a prior turn, false otherwise
+        
+        Rules:
+        - Greetings / chit-chat → conversational, needs_context: false
+        - Questions about facts / documents → factual, needs_context: true
+        - "you said earlier" / pronouns like "it" / "that" → follow_up, needs_history: true
+        - Personal data, health, finance → sensitive
+        Return ONLY the JSON object. No explanation.
+        """
+    
+    response = llm.invoke([
+        SystemMessage(content=system),
+        HumanMessage(content=query)
+    ])
+    raw = response.content.strip()
+    # Strip markdown fences if present
+    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
+
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        parsed = {}
+
+    return {
+        "intent": "factual",
+        "needs_context": True,
+        "needs_history": False,
+        **parsed
+    }
+
+# query= "How do I top-up the value of my card when I am abroad"
+# trace = []
+# intent_result = analyse_intent(query)
+# trace.append({
+#         "step": "intent_analysis",
+#         "result": intent_result
+#     })
+# print(f"[Intent] {intent_result}")

agentic_workflow/pii.py

@@ -0,0 +1,30 @@
+from presidio_analyzer import AnalyzerEngine, PatternRecognizer, Pattern
+from presidio_analyzer.nlp_engine import NlpEngineProvider
+from presidio_anonymizer import AnonymizerEngine
+
+_provider = NlpEngineProvider(nlp_configuration={
+    "nlp_engine_name": "spacy",
+    "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}]
+})
+
+analyzer  = AnalyzerEngine(nlp_engine=_provider.create_engine())
+anonymizer = AnonymizerEngine()
+
+analyzer.registry.add_recognizer(PatternRecognizer(
+    supported_entity="ACCOUNT_NUMBER",
+    patterns=[Pattern(name="account_number", regex=r"\d{3}-\d{3}-\d{3}", score=0.8)],
+    context=["account", "acct", "num", "number", "no.", "#"]
+))
+
+_ENTITIES = ["ACCOUNT_NUMBER", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "CREDIT_CARD"]
+
+# 2 funcs
+def check_pii(text: str) -> bool:
+    """Returns True if PII is detected — used by planner to block the query."""
+    results = analyzer.analyze(text=text, language="en", entities=_ENTITIES)
+    return len(results) > 0
+
+def scrub_pii(text: str) -> str:
+    """Replaces PII with placeholders — use this if you want to proceed with a sanitised query instead of blocking."""
+    results = analyzer.analyze(text=text, language="en", entities=_ENTITIES)
+    return anonymizer.anonymize(text=text, analyzer_results=results).text

agentic_workflow/planner.py

@@ -0,0 +1,101 @@
+import json
+import chromadb
+from agentic_workflow.config import LLM_MODEL
+from langchain_ollama import ChatOllama
+from dataclasses import dataclass, field
+
+llm = ChatOllama(model=LLM_MODEL, temperature=0)
+
+@dataclass
+class PlanStep:
+    name: str       # e.g. "retrieval", "pii_check"
+    enabled: bool   # whether this step should run
+    reason: str     # why it was enabled or skipped
+
+@dataclass
+class Plan:
+    steps: list[PlanStep] = field(default_factory=list)
+
+    def is_enabled(self, step_name: str) -> bool:
+        """Quick lookup — used by the pipeline to check if a step should run."""
+        return any(s.name == step_name and s.enabled for s in self.steps)
+
+    def summary(self) -> None:
+        """Prints a readable breakdown of the plan."""
+        print("\n── Execution Plan ──────────────────────────")
+        for s in self.steps:
+            status = "RUN " if s.enabled else "SKIP"
+            print(f"  [{status}] {s.name:<20} → {s.reason}")
+        print("────────────────────────────────────────────\n")
+
+def create_plan(intent_result: dict) -> Plan:
+    """
+    Reads the intent analysis output and decides which steps
+    the pipeline should execute, with a reason for each decision.
+
+    Args:
+        intent_result: dict returned by analyse_intent(), e.g.
+            {
+                "intent": "factual",
+                "needs_context": True,
+                "needs_history": False
+            }
+
+    Returns:
+        Plan object with a PlanStep for each possible action.
+    """
+    intent        = intent_result.get("intent", "factual")
+    needs_context = intent_result.get("needs_context", True)
+    needs_history = intent_result.get("needs_history", False)
+
+    steps = []
+
+    # PII check
+    # Always runs — sensitive queries must be gated regardless of intent
+    steps.append(PlanStep(
+        name="pii_check",
+        enabled=True,
+        reason="Always runs to protect against personal data leakage"
+    ))
+
+    # Vector retrieval
+    # Only runs for factual queries that need external knowledge
+    steps.append(PlanStep(
+        name="retrieval",
+        enabled=needs_context,
+        reason=(
+            "Query requires knowledge base context"
+            if needs_context
+            else f"Skipped — '{intent}' intent does not need retrieval"
+        )
+    ))
+
+    # Conversation history
+    # Only runs when the query references a prior turn
+    steps.append(PlanStep(
+        name="history",
+        enabled=needs_history,
+        reason=(
+            "Query appears to reference a prior turn"
+            if needs_history
+            else "Skipped — query is self-contained"
+        )
+    ))
+
+    # LLM generation
+    # Always runs — we always need a response
+    steps.append(PlanStep(
+        name="llm_generation",
+        enabled=True,
+        reason="Always runs to generate the final response"
+    ))
+
+    # Quality evaluation
+    # Always runs — every response should be checked before returning
+    steps.append(PlanStep(
+        name="quality_eval",
+        enabled=True,
+        reason="Always runs to verify response quality before returning"
+    ))
+
+    return Plan(steps=steps)

agentic_workflow/retriever.py

@@ -0,0 +1,56 @@
+# from chroma.chroma_client import knowledge_col, corrections_col
+# from chatbot_embed import embed
+# from agentic_workflow.config import TOP_K
+from sentence_transformers import SentenceTransformer
+import chromadb
+from pathlib import Path
+
+base_dir = Path(__file__).resolve().parent.parent
+embed_model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")
+db_path = base_dir / "chroma" / "chroma_db"
+client     = chromadb.PersistentClient(path= str(db_path))
+
+print(client.list_collections()) 
+
+def retrieve_from_vector_db(query: str, collection_name: str = "bank_faq", k: int = 3) -> list[dict]:
+    """
+    Embeds the query using the same model used during ingestion,
+    queries ChromaDB, and returns chunks with their similarity scores.
+    """
+
+    collection = client.get_collection(name=collection_name)
+
+    # Embed the query with the same model used at ingestion
+    query_embedding = embed_model.encode(query).tolist()
+
+    results = collection.query(
+        query_embeddings=[query_embedding],
+        n_results=k,
+        include=["documents", "metadatas", "distances"]
+    )
+
+    # ChromaDB returns distances (lower = more similar), convert to similarity score
+    chunks = []
+    for doc, metadata, distance in zip(
+        results["documents"][0],
+        results["metadatas"][0],
+        results["distances"][0]
+    ):
+        chunks.append({
+            "text": doc,
+            "metadata": metadata,
+            "similarity_score": round(1 - distance, 4)   # convert distance → similarity
+        })
+
+    return chunks
+
+results = retrieve_from_vector_db(query="What are the requirements to open a bank account?", k=3)
+
+# print("Test 1: ")
+# for i, chunk in enumerate(results, 1):
+#     print(f"  Chunk {i}:")
+#     print(f"    Score:    {chunk['similarity_score']}")
+#     print(f"    Metadata: {chunk['metadata']}")
+#     print(f"    Text:     {chunk['text'][:100]}...")
+# print()
+

app.py

@@ -0,0 +1,98 @@
+import streamlit as st
+from chroma.chatbot_retriever import ask
+from chroma.chatbot_chroma_db import create_new_chat, delete_chat, load_chats_from_db
+
+# Initialize session state
+if "chats" not in st.session_state:
+    loaded = load_chats_from_db()
+    if loaded:
+        st.session_state.chats = loaded
+    else:
+        initial_chat_id = create_new_chat()
+        st.session_state.chats = {"Chat 1": {"messages": [], "chat_id": initial_chat_id}}
+
+if "active_chat" not in st.session_state:
+    st.session_state.active_chat = "Chat 1"
+
+
+# st.write(list(st.session_state.chats.keys()))
+
+# print(st.session_state.active_chat)
+
+
+# sidebar
+with st.sidebar:
+    st.title("Bank Chatbot")
+    # New Chat button
+    if st.button("+ New Chat", use_container_width=True):
+        new_name = f"Chat {len(st.session_state.chats) + 1}"
+        new_chat_id = create_new_chat()
+        st.session_state.chats[new_name] = {"messages": [], "chat_id": new_chat_id}
+        st.session_state.active_chat = new_name
+        st.rerun()
+
+    st.divider()
+
+    # List all chats as buttons
+    for chat_name in list(st.session_state.chats.keys()):  # list() avoids mutation during loop
+        col1, col2 = st.columns([4, 1])
+
+        with col1:
+            if st.button(chat_name, key=f"select_{chat_name}", use_container_width=True):
+                st.session_state.active_chat = chat_name
+                st.rerun()
+
+        with col2:
+            if st.button("🗑", key=f"delete_{chat_name}"):
+                delete_chat(st.session_state.chats[chat_name]["chat_id"])
+
+                del st.session_state.chats[chat_name]
+
+                # if deleted chat was active, switch to another
+                if st.session_state.active_chat == chat_name:
+                    remaining = list(st.session_state.chats.keys())
+                    if remaining:
+                        st.session_state.active_chat = remaining[0]
+                    else:
+                        # no chats left, create a fresh one
+                        new_chat_id = create_new_chat()
+
+                        st.session_state.chats["Chat 1"] = {
+                            "messages": [],
+                            "chat_id": new_chat_id
+                        }
+                        st.session_state.active_chat = "Chat 1"
+
+                st.rerun()
+
+st.header(st.session_state.active_chat)
+
+active = st.session_state.active_chat
+# st.write(st.session_state.chats)
+# Display history for the active chat
+for msg in st.session_state.chats[active]["messages"]:
+    with st.chat_message(msg["role"]):
+        st.write(msg["content"])
+
+if prompt := st.chat_input("Ask me anything..."):
+    # if "chat_id" not in st.session_state:
+    #     st.session_state.chat_id = create_new_chat()
+
+    active = st.session_state.active_chat
+    chat_id = st.session_state.chats[active]["chat_id"]
+    
+    # saves user message
+    st.session_state.chats[active]["messages"].append({"role": "user", "content": prompt})
+    
+    # with st.chat_message("user"):
+    #     st.write(prompt)
+
+    # add response from RAG chain, passing session chat history
+    # with st.chat_message("assistant"):
+    with st.spinner("Thinking..."):
+        response = ask(prompt, chat_id = chat_id, chat_name=active)
+    # st.write(response['answer'])
+
+    
+    st.session_state.chats[active]["messages"].append({"role": "assistant", "content": response})
+    st.rerun()

data_preparation/chatbot_data_ingest.py

@@ -0,0 +1,66 @@
+from chonkie import SemanticChunker
+from sentence_transformers import SentenceTransformer
+from pathlib import Path
+import pandas as pd
+
+data_dir = Path("../datasets")
+train_faq_df = pd.read_csv(data_dir / "processed" / "bank_faq" /"train_faq.csv")
+
+# Initialize with an embedding model
+chunker = SemanticChunker(
+    embedding_model="multi-qa-mpnet-base-dot-v1",
+    chunk_size=200
+)
+
+chunked_docs = []
+for idx, row in train_faq_df.iterrows():
+    question = row["Question"]
+    ans = row["Answer"]
+    domain   = row["Class"]
+
+    if len(ans.split()) < 200:
+        # short — keep as single doc, embed q+a together
+        chunked_docs.append({
+            "text":      row["QA_pair"],  # for embedding
+            "question":  question,
+            "answer":    ans,
+            "domain":    domain,
+            "source_id": str(idx),
+            "chunk_num": "0"
+        })
+    else:
+        # long — chunk only the answer
+        chunks = chunker.chunk(ans)
+        for chunk_num, chunk in enumerate(chunks):
+            chunked_docs.append({
+                "text":      f"Q: {question} A: {chunk.text}",  # question anchors every chunk
+                "question":  question,
+                "answer":    chunk.text,
+                "domain":    domain,
+                "source_id": str(idx),
+                "chunk_num": str(chunk_num)
+            })
+
+print(f"Total chunks: {len(chunked_docs)}") 
+
+print(chunked_docs[0])
+
+
+
+model = SentenceTransformer("multi-qa-mpnet-base-dot-v1")
+
+texts = [doc["text"] for doc in chunked_docs]
+embeddings = model.encode(texts, batch_size=64, show_progress_bar=True)
+
+embed_faq_df = pd.DataFrame(chunked_docs)
+embed_faq_df["embedding"] = embeddings.tolist()
+
+embed_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "embed_faq.csv", index=False)
+print(embed_faq_df.shape)
+print(embed_faq_df.head())
+
+# for doc in chunked_docs:
+#     print(len(doc["text"]), "|", doc["text"][:80])
+
+# cleaned_faq_df["token_count"] = cleaned_faq_df["QA_pair"].apply(lambda x: len(x.split()))
+# print(cleaned_faq_df["token_count"].describe())

data_preparation/chatbot_data_kb.py

@@ -0,0 +1,24 @@
+import pandas as pd
+import json
+import ollama
+import re
+from tqdm import tqdm
+tqdm.pandas()
+from pathlib import Path
+
+data_dir = Path("../datasets")
+
+# faq qa pair
+train_faq_df = pd.read_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv")
+train_faq_df["QA_pair"] = "Q: " + train_faq_df["Question"] + "\nA: " + train_faq_df["Answer"]
+train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
+
+# train_faq_df = train_faq_df[["Question", "Answer", "QA_pair", "Class"]].rename(columns={
+#     "Question": "question",
+#     "Answer":   "answer",
+#     "Class": "domain"
+# })
+# train_faq_df["source"]   = "faq"
+# train_faq_df["industry"] = "banking"
+
+

data_preparation/chatbot_data_prep.py

@@ -0,0 +1,167 @@
+# data preparation for chatbot
+from pathlib import Path
+import pandas as pd
+import re
+from tqdm import tqdm
+from nltk.tokenize import word_tokenize
+import nltk
+from wordfreq import word_frequency, zipf_frequency
+try:
+    nltk.data.find("tokenizers/punkt")
+except LookupError:
+    nltk.download("punkt")
+from nltk.corpus import words
+
+nltk.download('words', quiet=True)
+from lingua import Language
+from lingua import LanguageDetectorBuilder
+from gibberish_detector import detector
+tqdm.pandas()
+from sklearn.model_selection import train_test_split
+
+# load datasets
+data_dir = Path("../datasets")
+
+faq_df = pd.read_csv(data_dir / "raw" / "BankFAQs.csv")
+support_df = pd.read_csv(data_dir / "raw" / "customer_support_data.csv")
+
+print("FAQ dataset")
+print(faq_df.head())
+print("Customer Support dataset")
+print(support_df.head())
+
+eng_words = set(word.lower() for word in words.words())
+
+# load the gibberish detection model
+Detector = detector.create_from_model('gibberish-detector.model')
+
+stopwords = {
+    "ho", "rahi", "hai", "ke", "mein", "raha", "hoon", "kar"
+}
+
+# cleaning customer support dataset
+def clean_text(text):
+    # handle missing values
+    if pd.isna(text):
+        return ""
+
+    text = text.replace("’", "'").replace("’", "'")
+    # text = re.sub(r"[^a-zA-Z0-9'\s-]", ' ', text)
+    # text = re.sub(r'\s+', ' ', text).strip()
+
+    # split into phrases by punctuation
+    phrases = re.split(r'[.!?,;:/]', text)
+    # print(phrases)
+    
+    clean_phrases = []
+    total_removed = 0
+    
+    for phrase in phrases:
+        phrase = phrase.strip()
+        if not phrase:
+            continue
+
+        tokens = re.findall(r"[a-zA-Z0-9]*[0-9][a-zA-Z][a-zA-Z0-9]*|[a-zA-Z]+(?:'[a-zA-Z]+)*|[0-9]+(?:-[0-9]+)*", phrase)
+        
+        real_words = []
+        removed_words = []
+    
+        for token in tokens:
+            if re.fullmatch(r"[a-zA-Z0-9]*[0-9][a-zA-Z]+", token):
+                real_words.append(token)
+                continue
+
+            if re.fullmatch(r"[0-9]+(?:-[0-9]+)*", token):
+                real_words.append(token)
+                continue
+
+            token_lower = token.lower()
+
+            if token_lower in stopwords:
+                removed_words.append(token)
+                continue
+
+            if Detector.is_gibberish(token_lower):
+                # print(token)
+                removed_words.append(token)
+                continue
+
+            if token_lower in {"a", "i"}:
+                real_words.append(token)
+                continue
+            
+            # contractions - keep directly without English check
+            if "'" in token_lower:
+                real_words.append(token)
+                continue
+
+            if token_lower in eng_words or zipf_frequency(token_lower, "en") >= 2.0:
+                real_words.append(token)
+                continue
+
+            removed_words.append(token)
+
+        # print("Real Words: ", real_words)
+        # print(f"  Removed words : {removed_words} ({len(removed_words)} removed)")
+
+        if not real_words:
+            continue
+
+        # keep phrases when at least a reasonable portion of tokens are English-like
+        # if len(real_words) / len(tokens) < 0.25:
+        #     continue
+
+        if len(removed_words) >= len(real_words):
+            continue
+        
+        total_removed += len(removed_words)
+        clean_phrases.append(" ".join(real_words))
+
+    # print(f"\nTotal words removed: {total_removed}")
+    return " ".join(clean_phrases)
+
+# cleaned_support_df = support_df[support_df["language"] == "en"][["conv_id", "turn_index", "role", "text", "industry", "product", "outcome", "issue_type", "overall_urgency"]].copy()
+# cleaned_support_df["text"] = cleaned_support_df["text"].progress_apply(clean_text)
+
+# cleaned_support_df = cleaned_support_df[cleaned_support_df["text"].str.strip() != ""]
+# cleaned_support_df = cleaned_support_df.dropna(subset=["text"])
+# cleaned_support_df.to_csv(data_dir / "processed" / "customer_support" / "cleaned_support_text.csv", index=False)
+
+
+
+# row_text = support_df.loc[support_df["language"] == "en", "text"].iloc[4]
+
+# cleaned_text = clean_text(row_text)
+# print(row_text)
+# print(cleaned_text)
+
+
+# cleaning bank faqs
+faq_df = faq_df.drop_duplicates(subset=["Question", "Answer"])
+
+def clean_text_faq(text):
+    if not isinstance(text, str):
+        return text
+    text = re.sub(r"[\r\n\t]+", " ", text)             # normalise line breaks
+    text = re.sub(r"[^\x00-\x7F]+", "", text)
+    text = re.sub(r"<<\s*>>", "", text)  
+    text = re.sub(r" {2,}", " ", text)                 # collapse spaces
+    return text.strip()
+
+for col in ["Question", "Answer"]:
+    faq_df[col] = faq_df[col].progress_apply(clean_text_faq)
+faq_df.to_csv(data_dir / "processed" / "bank_faq" / "cleaned_faq.csv", index=False)
+
+# split dataset for train/test
+train_faq_df, test_faq_df = train_test_split(
+    faq_df,
+    test_size=0.2,
+    stratify=faq_df["Class"],
+    random_state=42
+)
+
+print(len(train_faq_df))
+print(len(test_faq_df))
+
+train_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "train_faq.csv", index=False)
+test_faq_df.to_csv(data_dir / "processed" / "bank_faq" / "test_faq.csv", index=False)

data_preparation/gibberish-detector.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dafd84bd85ed88afa837ebc7121210e97eeb61053513f659800f495a2e813e63
+size 26296

datasets/processed/bank_faq/embed_faq.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20343606ef18ffe9b32cc95cf9690a9da8bf8fa9eb926001bf9d6c252c641ca1
+size 21668106

datasets/processed/customer_support/cleaned_support_text.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1558e646f928077b899259269f269eb88df382b2c2d2c4172be278bb0fe38c22
+size 35090455

datasets/raw/customer_support_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5951c6349f8bb9a73410daea8ab635fbaa498f1d920eb57bd738c45170ec6360
+size 628177158

eval_pipeline.py

@@ -0,0 +1,432 @@
+from pathlib import Path
+import pandas as pd
+# import math, random
+from sentence_transformers import SentenceTransformer
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_ollama import ChatOllama
+from langchain_core.messages import HumanMessage, SystemMessage
+import chromadb
+from langchain_chroma import Chroma
+from sklearn.metrics.pairwise import cosine_similarity
+from chroma.chatbot_retriever import ask
+from tqdm import tqdm
+import random
+import json
+import numpy as np
+import re
+
+model      = HuggingFaceEmbeddings(model_name="multi-qa-mpnet-base-dot-v1")
+client     = chromadb.PersistentClient(path="chroma/chroma_db")
+data_dir = Path("datasets")
+llm = ChatOllama(model="llama3.2", temperature=0)
+
+vectorstore = Chroma(
+    client = client,
+    collection_name = "bank_faq",
+    embedding_function = model
+)
+
+retriever = vectorstore.as_retriever(
+    search_type = "similarity",
+    search_kwargs = {"k": 3}
+)
+   
+
+# test_docs = vectorstore.similarity_search("what are the fees", k=3)
+# print("Search results:", test_docs)
+
+# test_docs_scores = vectorstore.similarity_search_with_score("what are the fees", k=3)
+# print("With scores:", test_docs_scores)
+
+from difflib import SequenceMatcher
+
+# def is_correct(predicted: str, expected: str, threshold=0.6) -> bool:
+#     ratio = SequenceMatcher(None, predicted.lower(), expected.lower()).ratio()
+#     return ratio >= threshold
+
+def llm_judge(query, expected, predicted, llm):
+    prompt = f"""
+    You are evaluating a question answering system.
+    
+    Question: {query}
+    Expected Answer: {expected}
+    Predicted Answer: {predicted}
+    
+    IMPORTANT RULES:
+    - Focus on whether the MEANING and KEY INFORMATION is the same
+    - As long as most of the words matches with the expected answer, do not score as 0.0
+    - Ignore differences in formatting (bullet points vs numbered list vs plain text)
+    - Ignore minor wording differences as long as the information is the same
+    - Mark as correct if the predicted answer covers all the key points in the expected answer
+    - Mark as incorrect ONLY if key information is missing or wrong
+    
+    Respond ONLY in this JSON format, no other text:
+    - "correct": True if the meaning of the response matches with the expected answer, otherwise False
+    - "score": 0.0 - 1.0
+    - "reason": one-sentence explanation
+    """
+    
+    response = llm.invoke(prompt)
+
+    # If response has a 'content' attribute (e.g. an AIMessage object), use response.content. Otherwise, use response itself.
+    # text = str(getattr(response, "content", response))
+    # print(text)
+    # Remove Markdown code block markers and trim any leading/trailing whitespace.
+    # text = re.sub(r"```json|```", "", text).strip()
+    # Search for the first JSON object in the text.
+    # \{.*\} matches everything between the first '{' and the last '}'
+    raw = response.content.strip()
+    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
+   
+ 
+
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError as e:
+        print(f"JSON parse failed: {e}\nRaw text: {raw!r}")
+        parsed = {}
+
+    return {
+        "correct": False,
+        "score": 0.0,
+        "reason": "Predicted does not match with expected",
+        **parsed
+    }
+
+# from chroma.chatbot_retriever import ask
+train_dataset = pd.read_csv(data_dir/"processed"/"bank_faq" /"train_faq.csv")
+test_dataset = pd.read_csv(data_dir/"processed"/"bank_faq" /"test_faq.csv")
+
+train_dataset = train_dataset.rename(columns={
+    "Question": "query",
+    "Answer":   "expected_answer"
+}).to_dict(orient="records")
+
+# test_dataset = test_dataset.rename(columns={
+#     "Question": "query",
+#     "Answer":   "expected_answer"
+# }).to_dict(orient="records")
+
+def retrieval_success_check(expected, retrieved_text, threshold=0.6):
+    # embed both the expected answer and the retrieved docs into vector space
+    expected_vec = model.embed_query(expected)
+    retrieved_vec = model.embed_query(retrieved_text)
+
+    # measure how semantically similar they are (0 = no match, 1 = identical)
+    score = cosine_similarity([expected_vec], [retrieved_vec])[0][0]
+
+    # consider retrieval successful if similarity meets the threshold
+    return score >= threshold
+
+# train_dataset = [
+#     {
+#         "query":            item["question"],
+#         "expected_answer":  flatten_answer(item["answer"]),
+#         "relevant_doc_ids": extract_doc_ids(item["answer"])
+#     }
+#     for item in train_db
+# ]
+
+# test_dataset = [
+#     {
+#         "query":            row["Question"],
+#         "expected_answer":  row["Answer"],
+#         "relevant_doc_ids": []        # unknown for unseen data
+#     }
+#     for _, row in test_df.iterrows()
+# ]
+
+# def evaluate(dataset: list[dict], split: str = "", sample: int = None) -> float:
+#     data = random.sample(dataset, sample) if sample else dataset
+#     correct = 0
+#     for item in tqdm(data, desc=f"Evaluating {split}", unit="q"):
+#         predicted = ask(query=item["query"], persist=False)
+#         if is_correct(predicted, item["expected_answer"]):
+#             correct += 1
+#     return correct / len(data)
+
+# train_accuracy = evaluate(train_dataset, split="train", sample=200)
+# test_accuracy  = evaluate(test_dataset,  split="test",  sample=100)
+
+# print(f"Train accuracy: {train_accuracy:.2%}")
+# print(f"Test accuracy:  {test_accuracy:.2%}")
+
+# import matplotlib.pyplot as plt
+# import seaborn as sns 
+# labels     = ["Train", "Test"]
+# accuracies = [train_accuracy, test_accuracy]
+
+# plt.figure(figsize=(6, 4))
+# plt.bar(labels, accuracies, color=["steelblue", "coral"], width=0.4)
+# plt.ylim(0, 1)
+# plt.ylabel("Accuracy")
+# plt.title("Chatbot Accuracy — Train vs Test")
+
+# for i, v in enumerate(accuracies):
+#     plt.text(i, v + 0.02, f"{v:.1%}", ha="center", fontweight="bold")
+
+# plt.tight_layout()
+# plt.savefig("accuracy.png")
+# plt.show()
+
+
+# def evaluate(dataset: list[dict], split: str = "", sample: int = None):
+#     data = random.sample(dataset, sample) if sample else dataset
+
+#     results = []
+
+#     correct           = 0
+#     hallucinated      = 0
+#     retrieval_failures = 0
+#     total_recall      = 0
+#     total_precision   = 0
+
+#     for item in tqdm(data, desc=f"Evaluating {split}", unit="q"):
+
+#         query              = item["query"]
+#         expected           = item["expected_answer"]
+#         relevant_doc_ids   = item["relevant_doc_ids"]   # ← from your tagged dataset
+
+#         # run chatbot
+#         predicted = ask(query=query, persist=False)
+
+#         # --- retrieval ---
+#         retrieved_docs = retriever.invoke(query)
+#         recall, precision = retrieval_metrics(retrieved_docs, relevant_doc_ids, k=K)
+
+#         retrieval_success = recall > 0   # at least one relevant chunk retrieved
+
+#         total_recall    += recall
+#         total_precision += precision
+
+#         if not retrieval_success:
+#             retrieval_failures += 1
+
+#         # --- correctness ---
+#         correct_flag = is_correct(predicted, expected)
+
+#         # --- hallucination ---
+#         # hallucination = retrieval worked but LLM still got it wrong
+#         # (if retrieval failed, it's a retrieval problem, not hallucination)
+#         hallucinated_flag = retrieval_success and not correct_flag
+
+#         if correct_flag:
+#             correct += 1
+
+#         if hallucinated_flag:
+#             hallucinated += 1
+
+#         results.append({
+#             "query":            query,
+#             "expected":         expected,
+#             "predicted":        predicted,
+#             "relevant_doc_ids": relevant_doc_ids,
+#             "retrieved_doc_ids": [doc.metadata["doc_id"] for doc in retrieved_docs[:K]],
+#             "retrieval_success": retrieval_success,
+#             f"recall@{K}":      recall,
+#             f"precision@{K}":   precision,
+#             "correct":          correct_flag,
+#             "hallucinated":     hallucinated_flag
+#         })
+
+#     metrics = {
+#         "split":                 split,
+#         "accuracy":              correct / len(data),
+#         "hallucination_rate":    hallucinated / len(data),
+#         "retrieval_failure_rate": retrieval_failures / len(data),
+#         f"recall@{K}":           total_recall / len(data),
+#         f"precision@{K}":        total_precision / len(data),
+#         "samples":               len(data)
+#     }
+
+#     return metrics, results
+
+def measure_hallucination(query: str, response: str, chunks: list[dict], llm) -> dict:
+    """
+    Uses the LLM to check whether the response contains claims
+    not supported by the retrieved chunks.
+ 
+    Returns:
+      - hallucinated: bool
+      - confidence:   "high" | "medium" | "low"
+      - reason:       short explanation
+    """
+    context = "\n".join(c["text"] for c in chunks) if chunks else "No context retrieved."
+ 
+    system = """
+        You are a hallucination detector. Given a context, a query, and a response, 
+        determine if the response contains information NOT supported by the context.
+        Return ONLY valid JSON:
+        - "hallucinated": true if the response contains unsupported claims, false otherwise
+        - "confidence": "high", "medium", or "low"
+        - "reason": one-sentence explanation
+        
+        Return ONLY the JSON object. No explanation.
+        """
+ 
+    result = llm.invoke([
+        SystemMessage(content=system),
+        HumanMessage(content=(
+            f"Context:\n{context}\n\n"
+            f"Query: {query}\n\n"
+            f"Response: {response}"
+        ))
+    ])
+ 
+    raw = result.content.strip()
+    raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw).strip()
+ 
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        parsed = {}
+ 
+    return {
+        "hallucinated": True,
+        "confidence":   "low",
+        "reason":       "Hallucination check failed",
+        **parsed
+    }
+
+
+FALLBACK_PHRASES = [
+    "I don't have information on that.",
+]
+
+def evaluate(dataset: list[dict], split: str = "", sample: int = None):
+    data = random.sample(dataset, sample) if sample else dataset
+
+    results = []
+
+    correct = 0
+    hallucinated = 0
+    retrieval_failures = 0
+
+    for item in tqdm(data, desc=f"Evaluating {split}", unit="q"):
+
+        query = item["query"]
+        expected = item["expected_answer"]
+
+        # run chatbot
+        predicted = ask(query=query, persist=False)
+
+        # Score-aware retrieval
+        docs_with_scores = vectorstore.similarity_search_with_score(query, k=3)
+        # print("Docs: ", docs_with_scores)
+        # --- retrieval check (simple proxy) ---
+        # retrieved_docs = retriever.invoke(query) 
+
+        retrieved_docs = [doc for doc, _ in docs_with_scores]
+        confidence_scores = [round(1 / (1 + score), 4) for _, score in docs_with_scores]
+        avg_confidence = round(sum(confidence_scores) / len(confidence_scores), 4) if confidence_scores else 0.0
+
+        # retrieved_text = " ".join(retrieved_docs).lower()
+        retrieved_text = " ".join([doc.page_content for doc in retrieved_docs]).lower()
+        expected_in_context = expected.lower() in retrieved_text
+
+        retrieval_success = retrieval_success_check(expected, retrieved_text)
+
+        # --- correctness ---
+        # correct_flag = is_correct(predicted, expected)
+        
+        is_fallback = any(phrase.lower() in predicted.lower() for phrase in FALLBACK_PHRASES)
+
+        if is_fallback:
+            judgement = {
+                "correct": False,
+                "score": 0.0,
+                "reason": "Model returned a fallback response"
+            }
+            hallucinated_flag = False
+            hallucination_detail = {
+                "hallucinated": False,
+                "confidence": "high",
+                "reason": "Model does not know the answer"
+            }
+        else:
+            judgement = llm_judge(query, expected, predicted, llm)
+
+            if judgement["correct"]:
+                hallucinated_flag = False
+                hallucination_detail = {
+                    "hallucinated": False,
+                    "confidence": "high",
+                    "reason": "Response is correct"
+                }
+                correct += 1
+            else:
+                chunks = [
+                    {
+                        "text":             doc.page_content,
+                        "similarity_score": score
+                    }
+                    for doc, score in docs_with_scores
+                ]
+                hallucination_detail  = measure_hallucination(query, predicted, chunks, llm)
+                hallucinated_flag     = hallucination_detail["hallucinated"]
+            
+        # hallucinated_flag = not expected_in_context and not correct_flag and not is_fallback
+
+        # if correct_flag:
+        #     correct += 1
+
+        if hallucinated_flag:
+            hallucinated += 1
+
+        if not retrieval_success:
+            retrieval_failures += 1
+
+        results.append({
+            "query": query,
+            "expected": expected,
+            "predicted": predicted,
+            "retrieval_success": bool(retrieval_success),
+            "correct": judgement["correct"],
+            "score": judgement["score"],
+            "reason": judgement["reason"],
+            "is_fallback": is_fallback, 
+            "hallucination_detail": hallucination_detail,
+            "confidence_scores": confidence_scores, 
+            "avg_confidence":    avg_confidence
+        })
+
+    metrics = {
+        "split": split,  
+        "accuracy": correct / len(data),
+        "hallucination_rate": hallucinated / len(data),
+        "retrieval_failure_rate": retrieval_failures / len(data),
+        "samples": len(data)
+    }
+
+    return metrics, results
+
+def save_results(metrics, results, filename="eval_results.json"):
+    output = {
+        "metrics": metrics,
+        "detailed_results": results
+    }
+
+    with open(filename, "w") as f:
+        json.dump(output, f, indent=2)
+
+train_metrics, train_results = evaluate(train_dataset, "train", sample=200)
+# test_metrics, test_results   = evaluate(test_dataset, "test", sample=20)
+
+# Bucket your failures:
+# retrieval_failed = len([x for x in test_results if not x["retrieval_success"]])
+# retrieval_ok_but_wrong = len([x for x in test_results if x["retrieval_success"] and not x["correct"]])
+# fallbacks = len([x for x in test_results if x["is_fallback"]])
+
+# print(retrieval_failed)
+# print(retrieval_ok_but_wrong)
+# print(fallbacks)
+
+save_results(train_metrics, train_results, "train_eval.json")
+# save_results(test_metrics, test_results, "test_eval.json")
+
+
+
+
+
+
+

requirements.txt

@@ -0,0 +1,23 @@
+chonkie==1.6.7
+chromadb==1.5.9
+gibberish_detector==0.1.1
+huggingface_hub==1.10.2
+huggingface_hub==1.16.1
+langchain_chroma==1.1.0
+langchain_classic==1.0.8
+langchain_community==0.4.2
+langchain_core==1.4.8
+langchain_huggingface==1.2.2
+langchain_ollama==1.1.0
+lingua==4.16.2
+nltk==3.9.4
+numpy==2.5.1
+pandas==3.0.3
+presidio_analyzer==2.2.363
+presidio_anonymizer==2.2.363
+scikit_learn==1.9.0
+sentence_transformers==5.5.1
+streamlit==1.57.0
+tqdm==4.67.1
+tqdm==4.67.3
+wordfreq==3.1.1