Upload 12 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Datasets/Dataset.json filter=lfs diff=lfs merge=lfs -text
+Datasets/flattened_drug_dataset_cleaned.csv filter=lfs diff=lfs merge=lfs -text
+Vectors/faiss_index.idx filter=lfs diff=lfs merge=lfs -text

Datasets/Dataset.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc38f7e5bfad6d7c2865ed7c94d483c8b9b887a47853e4a3c16ce957ce1f06a0
+size 35120734

Datasets/flattened_drug_dataset_cleaned.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0669d5d7366973a342a3cc35321366a02837c66ac5e7c28c3bf0569897db5b84
+size 31338099

Scripts/Answer_Generation.py

@@ -0,0 +1,100 @@
+"""
+Answer Generation Module for Retrieval-based Medical QA Chatbot
+=================================================================
+This module handles:
+1. Building prompts for LLMs
+2. Querying the Groq API with selected context
+3. Generating a final answer based on retrieved chunks
+"""
+
+from openai import OpenAI
+import os
+from Retrieval import Retrieval_averagedQP
+
+# -------------------------------
+# Groq API Client Setup
+# -------------------------------
+
+client = OpenAI(
+    api_key=os.environ.get("GROQ_API_KEY"),
+    base_url="https://api.groq.com/openai/v1"  
+)
+
+# -------------------------------
+# Function: Query Groq API
+# -------------------------------
+
+def query_groq(prompt, model="meta-llama/llama-4-scout-17b-16e-instruct", max_tokens=300):
+    """
+    Sends a prompt to Groq API and returns the generated response.
+    Parameters:
+        prompt (str): The text prompt for the model.
+        model (str): Model name deployed on Groq API.
+        max_tokens (int): Maximum tokens allowed in the output.
+    Returns:
+        str: Model-generated response text.
+    """
+    response = client.chat.completions.create(
+        model=model,
+        messages=[
+            {"role": "system", "content": "You are a biomedical assistant."},
+            {"role": "user", "content": prompt}
+        ],
+        temperature=0.7,
+        max_tokens=max_tokens
+    )
+    return response.choices[0].message.content.strip()
+
+# -------------------------------
+# Function: Build Prompt
+# -------------------------------
+
+def build_prompt(question, context):
+    """
+    Constructs a prompt for the model combining the user question and retrieved context.
+    Parameters:
+        question (str): User's question.
+        context (str): Retrieved relevant text chunks.
+    Returns:
+        str: Complete prompt text.
+    """
+    return f"""Strictly based on the following information, answer the question: {question}
+Do not explain the context, just provide a direct answer.
+Context:
+{context}
+"""
+
+# -------------------------------
+# Function: Answer Generation
+# -------------------------------
+
+def answer_generation(question, top_chunks, top_k=3):
+    """
+    Generates an answer based on retrieved top chunks.
+    Parameters:
+        question (str): User's question.
+        top_chunks (DataFrame): Retrieved top chunks with context.
+        top_k (int): Number of top chunks to use for answer generation.
+    Returns:
+        str: Final generated answer.
+    """
+    # Select top-k chunks
+    top_chunks = top_chunks.head(top_k)
+    print("[Answer Generation] Top chunks selected for generation.")
+
+    # Join context
+    context = "\n".join(top_chunks["chunk_text"].tolist())
+
+    # Build prompt and query Groq
+    prompt = build_prompt(question, context)
+    answer = query_groq(prompt)
+
+    return answer
+
+# -------------------------------
+# Example Usage (Uncomment to Test)
+# -------------------------------
+
+# question = "How is Aztreonam inhalation used?"
+# answer = answer_generation(question, top_chunks)
+# print("Generated Answer:", answer)

Scripts/Query_processing.py

@@ -0,0 +1,131 @@
+"""
+Query Processing Pipeline for Retrieval-based QA Chatbot
+========================================================
+
+This module handles:
+1. Query preprocessing
+2. Intent and sub-intent classification
+3. Named Entity Recognition (NER) using SciSpaCy
+
+"""
+
+import spacy
+import re
+from typing import List, Tuple
+
+# Load pre-trained SciSpaCy model for biomedical NER
+ner_model = spacy.load("en_core_sci_md")
+
+# -------------------------------
+# Rule-Based Intent Classification
+# -------------------------------
+
+def classify_intent(question: str) -> str:
+    """
+    Classify the user's query into a high-level intent based on keywords.
+    Replace this rule-based system with ML-based intent detection for scalability.
+
+    Parameters:
+        question (str): The user's question.
+
+    Returns:
+        str: One of ['description', 'before_using', 'proper_use', 'precautions', 'side_effects']
+    """
+    q = question.lower()
+
+    if re.search(r"\bwhat is\b|\bused for\b|\bdefine\b", q):
+        return "description"
+    elif re.search(r"\bbefore using\b|\bshould I tell\b|\bdoctor know\b", q):
+        return "before_using"
+    elif re.search(r"\bhow to\b|\bdosage\b|\btake\b|\binstructions\b", q):
+        return "proper_use"
+    elif re.search(r"\bprecaution\b|\bpregnan\b|\bbreastfeed\b|\brisk\b", q):
+        return "precautions"
+    elif re.search(r"\bside effect\b|\badverse\b|\bnausea\b|\bdizziness\b", q):
+        return "side_effects"
+    else:
+        return "description"  # default fallback
+
+
+# -------------------------------
+# Subsection Classification
+# -------------------------------
+
+def classify_subsection(question: str) -> str:
+    """
+    Identify more granular subtopics within each main intent.
+
+    Parameters:
+        question (str): The user's question.
+
+    Returns:
+        str: Sub-intent such as 'more common', 'incidence not known', etc.
+    """
+    q = question.lower()
+
+    if re.search(r"\bcommon side effects\b|\busual symptoms\b", q):
+        return "more common"
+    elif re.search(r"\bunknown\b|\brare\b|\bincidence\b", q):
+        return "incidence not known"
+    elif re.search(r"\bchildren\b|\bpediatric\b|\bkids\b", q):
+        return "pediatric"
+    elif re.search(r"\bbreastfeed\b|\bnursing\b|\blactation\b", q):
+        return "breastfeeding"
+    elif re.search(r"\belderly\b|\bgeriatric\b", q):
+        return "geriatric"
+    elif re.search(r"\binteract\b|\bcombination\b|\bcontraindications\b", q):
+        return "drug interactions"
+    else:
+        return ""
+
+
+# -------------------------------
+# Named Entity Extraction
+# -------------------------------
+
+def extract_entities_spacy(question: str) -> List[str]:
+    """
+    Use SciSpaCy NER model to extract biomedical entities.
+
+    Parameters:
+        question (str): User query.
+
+    Returns:
+        List[str]: Unique list of extracted entities.
+    """
+    doc = ner_model(question)
+    return list(set(ent.text for ent in doc.ents))
+
+
+# -------------------------------
+# Query Preprocessing Wrapper
+# -------------------------------
+
+def preprocess_query(raw_query: str) -> Tuple[Tuple[str, str], List[str]]:
+    """
+    Main preprocessing function that extracts:
+    - Intent
+    - Subsection
+    - Named Entities
+
+    Parameters:
+        raw_query (str): The raw user question.
+
+    Returns:
+        Tuple[Tuple[str, str], List[str]]: ((intent, sub_intent), list of entities)
+    """
+    try:
+        intent = classify_intent(raw_query)
+        sub_intent = classify_subsection(raw_query)
+        entities = extract_entities_spacy(raw_query)
+
+        if not entities:
+            print("[NER fallback] No entities found. Using raw query.")
+            return (intent or "", sub_intent or ""), []
+
+        print(f"[Query Processed] Intent = {intent} | Subsection = {sub_intent} | Entities = {entities}")
+        return (intent or "", sub_intent or ""), entities
+
+    except Exception as e:
+        print(f"[Preprocessing failed] {e}")
+        return ("", ""), []

Scripts/Retrieval.py

@@ -0,0 +1,171 @@
+"""
+Retrieval and FAISS Embedding Module for Medical QA Chatbot
+============================================================
+
+This module handles:
+1. Embedding documents
+2. Building and saving FAISS index
+3. Retrieval with initial FAISS search + reranking using BioBERT similarity
+"""
+
+import faiss
+import pandas as pd
+import numpy as np
+import torch
+from sentence_transformers import SentenceTransformer, util
+from sklearn.preprocessing import normalize
+from Query_processing import preprocess_query
+import os
+
+# -------------------------------
+# File Paths
+# -------------------------------
+
+# Get the project root directory (one level up from script_dir)
+project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+# Absolute paths for dataset and index files
+csv_path = os.path.join(project_root, 'Datasets', 'flattened_drug_dataset_cleaned.csv')
+faiss_index_path = os.path.join(project_root, 'Vectors', 'faiss_index.idx')
+doc_metadata_path = os.path.join(project_root, 'Vectors', 'doc_metadata.pkl')
+doc_vectors_path = os.path.join(project_root, 'Vectors', 'doc_vectors.npy')
+
+# Load the dataset
+df = pd.read_csv(csv_path).dropna(subset=['chunk_text'])
+
+# -------------------------------
+# Model Initialization
+# -------------------------------
+
+fast_embedder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+biobert = SentenceTransformer('pritamdeka/BioBERT-mnli-snli-scinli-scitail-mednli-stsb')
+
+# -------------------------------
+# Function: Embed and Build FAISS Index
+# -------------------------------
+
+def Embed_and_FAISS():
+    """
+    Embeds the drug dataset and builds a FAISS index for fast retrieval.
+    Saves the index, metadata, and document vectors to disk.
+    """
+    print("Embedding document chunks using fast embedder...")
+
+    # Build full context strings
+    df['full_text'] = df.apply(lambda x: f"{x['drug_name']} | {x['section']} > {x['subsection']} | {x['chunk_text']}", axis=1)
+
+    full_texts = df['full_text'].tolist()
+    doc_embeddings = fast_embedder.encode(full_texts, convert_to_numpy=True, show_progress_bar=True)
+
+    # Normalize embeddings and build index
+    doc_embeddings = normalize(doc_embeddings, axis=1, norm='l2')
+    dimension = doc_embeddings.shape[1]
+    index = faiss.IndexFlatIP(dimension)
+    index.add(doc_embeddings)
+
+    # Save index and metadata
+    faiss.write_index(index, faiss_index_path)
+    df.to_pickle(doc_metadata_path)
+    np.save(doc_vectors_path, doc_embeddings)
+
+    print("FAISS index built and saved successfully.")
+
+# -------------------------------
+# Function: Retrieve with Context and Averaged Embeddings
+# -------------------------------
+
+def retrieve_with_context_averagedembeddings(query, top_k=10, predicted_intent=None, detected_entities=None, alpha=0.8):
+    """
+    Retrieve top chunks using FAISS followed by reranking with BioBERT similarity.
+
+    Parameters:
+        query (str): User query text.
+        top_k (int): Number of top results to retrieve.
+        predicted_intent (str, optional): Detected intent to adjust retrieval.
+        detected_entities (list, optional): List of named entities.
+        alpha (float): Weight for combining query and intent embeddings.
+
+    Returns:
+        pd.DataFrame: Retrieved chunks with metadata and reranked scores.
+    """
+    print(f"[Retrieval Pipeline Started] Query: {query}")
+
+    # Embed and normalize the query
+    query_vec = fast_embedder.encode([query], convert_to_numpy=True)
+
+    if predicted_intent:
+        intent_vec = fast_embedder.encode([predicted_intent], convert_to_numpy=True)
+        query_vec = normalize((alpha * query_vec + (1 - alpha) * intent_vec), axis=1)
+
+    # Load FAISS index and search
+    index = faiss.read_index(faiss_index_path)
+    D, I = index.search(query_vec, top_k)
+
+    df_meta = pd.read_pickle(doc_metadata_path)
+    retrieved_df = df_meta.loc[I[0]].copy()
+    retrieved_df['faiss_score'] = D[0]
+
+    # BioBERT reranking
+    query_emb = biobert.encode(query, convert_to_tensor=True)
+    chunk_embs = biobert.encode(retrieved_df['full_text'].tolist(), convert_to_tensor=True)
+    cos_scores = util.pytorch_cos_sim(query_emb, chunk_embs)[0]
+    reranked_idx = torch.argsort(cos_scores, descending=True)
+
+    # Boost scores based on intent, subsection match, or entity presence
+    results = []
+    for idx in reranked_idx:
+        idx = int(idx)
+        row = retrieved_df.iloc[idx]
+        score = cos_scores[idx].item()
+
+        section = row['section'][0] if isinstance(row['section'], tuple) else row['section']
+        subsection = row['subsection'][0] if isinstance(row['subsection'], tuple) else row['subsection']
+        if isinstance(predicted_intent, tuple):
+            predicted_intent = predicted_intent[0]
+
+        if predicted_intent and section.strip().lower() == predicted_intent.strip().lower():
+            score += 0.05
+        if predicted_intent and predicted_intent.lower() in subsection.strip().lower():
+            score += 0.03
+        if detected_entities:
+            if any(ent.lower() in row['chunk_text'].lower() for ent in detected_entities):
+                score += 0.1
+
+        results.append({
+            'chunk_id': row['chunk_id'],
+            'drug_name': row['drug_name'],
+            'section': row['section'],
+            'subsection': row['subsection'],
+            'chunk_text': row['chunk_text'],
+            'faiss_score': row['faiss_score'],
+            'semantic_similarity_score': score
+        })
+
+    return pd.DataFrame(results)
+
+# -------------------------------
+# Function: Retrieval Wrapper
+# -------------------------------
+
+def Retrieval_averagedQP(raw_query, intent, entities, top_k=10, alpha=0.8):
+    """
+    Wrapper to retrieve top-k chunks given a raw user query.
+
+    Parameters:
+        raw_query (str): The user query.
+        intent (str): Predicted intent from query processing.
+        entities (list): Detected biomedical entities.
+        top_k (int): Number of top results to return.
+        alpha (float): Weighting between query and intent embeddings.
+
+    Returns:
+        pd.DataFrame: Top retrieved chunks with scores.
+    """
+    results_df = retrieve_with_context_averagedembeddings(
+        raw_query,
+        top_k=top_k,
+        predicted_intent=intent,
+        detected_entities=entities,
+        alpha=alpha
+    )
+    return results_df[['chunk_id', 'drug_name', 'section', 'subsection', 'chunk_text', 'faiss_score', 'semantic_similarity_score']]

Scripts/demo.py

@@ -0,0 +1,58 @@
+"""
+Main Execution Script for Retrieval-based Medical QA Chatbot
+============================================================
+
+This script handles:
+1. Query preprocessing
+2. Information retrieval
+3. Answer generation
+"""
+
+import warnings
+warnings.filterwarnings("ignore", category=UserWarning)
+
+
+from dotenv import load_dotenv
+load_dotenv()
+
+
+from Query_processing import preprocess_query
+from Retrieval import Retrieval_averagedQP
+from Answer_Generation import answer_generation
+from Retrieval import Embed_and_FAISS
+
+# -------------------------------
+# Optional: Embed and Store FAISS Index
+# -------------------------------
+# Uncomment the below line to generate embeddings and build the FAISS index if not already done.
+# Embed_and_FAISS()
+
+# -------------------------------
+# Define User Question
+# -------------------------------
+
+Question = input("Enter your question: ")
+
+# -------------------------------
+# Step 1: Query Preprocessing
+# -------------------------------
+
+(intent, sub_intent), entities = preprocess_query(Question)
+
+# -------------------------------
+# Step 2: Retrieve Relevant Chunks
+# -------------------------------
+
+top_chunks = Retrieval_averagedQP(Question, intent, entities, top_k=10, alpha=0.8)
+
+# -------------------------------
+# Step 3: Answer Generation
+# -------------------------------
+
+Generated_answer = answer_generation(Question, top_chunks, top_k=3)
+
+# -------------------------------
+# Display Generated Answer
+# -------------------------------
+
+print("Generated Answer:", Generated_answer)

Vectors/doc_metadata.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:800157a95b50080634fdce730014af49a8e0cf01d2dbb484785b15936dc9abff
+size 53368209

Vectors/doc_vectors.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54da3cd890cf384fdc3b7abcd6ed5f840c0f53da30615fd417fc8256fd1b5ca
+size 70190720

Vectors/faiss_index.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d68a5ccb27c94e357ab12eec21d5d54d903949ae37648202643eb33387156b
+size 70190637