Upload demo.ipynb

notebooks/demo.ipynb

@@ -0,0 +1,1074 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "intro-markdown",
+   "metadata": {},
+   "source": [
+    "# RAG Pipeline Demo: Surgery Oral Board Simulation\n",
+    "\n",
+    "This notebook demonstrates the usage of the RAG (Retrieval-Augmented Generation) pipeline developed for simulating surgery oral board scenarios. It utilizes the custom Python modules located in the `src/` directory.\n",
+    "\n",
+    "**Pipeline Components:**\n",
+    "* `data_processing`: Loads and preprocesses raw case data from `.docx` files.\n",
+    "* `ClinicalCaseProcessor`: Creates embeddings for clinical cases.\n",
+    "* `ClinicalCaseRetriever`: Retrieves relevant cases based on semantic similarity to a query.\n",
+    "* `AnswerEvaluator`: Uses an LLM to evaluate user responses against expected answers.\n",
+    "* `OralExamSimulator`: Orchestrates the simulation flow."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "setup-markdown",
+   "metadata": {},
+   "source": [
+    "## 1. Setup\n",
+    "\n",
+    "Import necessary libraries and custom modules. Define constants and handle Hugging Face Hub authentication."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "setup-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2025-04-22 12:59:49.329544: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
+      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
+      "E0000 00:00:1745341189.621398  614864 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
+      "E0000 00:00:1745341189.717816  614864 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
+      "W0000 00:00:1745341190.668036  614864 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n",
+      "W0000 00:00:1745341190.668086  614864 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n",
+      "W0000 00:00:1745341190.668088  614864 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n",
+      "W0000 00:00:1745341190.668090  614864 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n",
+      "2025-04-22 12:59:50.675395: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import sys \n",
+    "import re\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from dotenv import load_dotenv\n",
+    "from huggingface_hub import login\n",
+    "from sklearn.metrics import ndcg_score\n",
+    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
+    "from IPython.display import display, Markdown\n",
+    "\n",
+    "# --- Add project root to sys.path ---\n",
+    "project_root = os.path.abspath(os.path.join(os.getcwd(), os.pardir))\n",
+    "if project_root not in sys.path:\n",
+    "    sys.path.append(project_root)\n",
+    "\n",
+    "# --- Custom Module Imports ---\n",
+    "from src.data_processing import process_all_cases, ClinicalCaseProcessor\n",
+    "from src.retriever import ClinicalCaseRetriever, DummyRetriever\n",
+    "from src.evaluator import AnswerEvaluator\n",
+    "from src.simulator import OralExamSimulator\n",
+    "from src.evaluation_utils import retrieval_metrics\n",
+    "from src.synthetic_generator import generate_synthetic_case, process_synthetic_data\n",
+    "\n",
+    "# --- Configuration ---\n",
+    "DATA_FOLDER = \"../data/\"\n",
+    "PROCESSED_DATA_PATH = \"../processed_clinical_cases\"\n",
+    "EVALUATOR_MODEL_ID = \"meta-llama/Llama-3.2-3B-Instruct\"\n",
+    "EMBEDDING_MODEL_ID = \"all-MiniLM-L6-v2\"\n",
+    "\n",
+    "# Hugging Face Hub Authentication (using .env file in project root)\n",
+    "dotenv_path = os.path.join(project_root, 'hf_login.env')\n",
+    "load_dotenv(dotenv_path=dotenv_path)\n",
+    "hf_key = os.getenv(\"HF_KEY\")\n",
+    "\n",
+    "# --- Set Cache Directory (Optional) ---\n",
+    "cache_dir = \"/scratch/krb3ym/models/cache\" # Keep your specific path\n",
+    "os.environ['HF_HOME'] = cache_dir\n",
+    "os.makedirs(cache_dir, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "load-data-markdown",
+   "metadata": {},
+   "source": [
+    "## 2. Load and Preprocess Raw Data\n",
+    "\n",
+    "Load the clinical cases from the `.docx` files into a pandas DataFrame using the utility function from `src.data_processing`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "load-data-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processing case files from: ../data/\n",
+      "Warning: Could not parse filename format: BTK_-_45A__Breast_Cancer.docx\n",
+      "Loaded 99 cases. Displaying head:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>case_id</th>\n",
+       "      <th>clinical_presentation</th>\n",
+       "      <th>turn_id</th>\n",
+       "      <th>question</th>\n",
+       "      <th>answer</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>85A</td>\n",
+       "      <td>Appendicitis Pediatrics</td>\n",
+       "      <td>1</td>\n",
+       "      <td>You were called to the emergency department to...</td>\n",
+       "      <td>Okay, well, I would begin by evaluating the pa...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>85A</td>\n",
+       "      <td>Appendicitis Pediatrics</td>\n",
+       "      <td>2</td>\n",
+       "      <td>The patient is febrile at 101.2 degrees, heart...</td>\n",
+       "      <td>Okay, does he have a positive psoas or a Rovsi...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>85A</td>\n",
+       "      <td>Appendicitis Pediatrics</td>\n",
+       "      <td>3</td>\n",
+       "      <td>Yes, both are positive.</td>\n",
+       "      <td>I'm concerned about acute appendicitis, but th...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>85A</td>\n",
+       "      <td>Appendicitis Pediatrics</td>\n",
+       "      <td>4</td>\n",
+       "      <td>Labs are in order. White blood cell count of 1...</td>\n",
+       "      <td>Well, this is diagnostic of acute appendicitis...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>85A</td>\n",
+       "      <td>Appendicitis Pediatrics</td>\n",
+       "      <td>5</td>\n",
+       "      <td>Are there any other therapeutic options for th...</td>\n",
+       "      <td>Yes, the patient has early acute appendicitis ...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  case_id    clinical_presentation  turn_id  \\\n",
+       "0     85A  Appendicitis Pediatrics        1   \n",
+       "1     85A  Appendicitis Pediatrics        2   \n",
+       "2     85A  Appendicitis Pediatrics        3   \n",
+       "3     85A  Appendicitis Pediatrics        4   \n",
+       "4     85A  Appendicitis Pediatrics        5   \n",
+       "\n",
+       "                                            question  \\\n",
+       "0  You were called to the emergency department to...   \n",
+       "1  The patient is febrile at 101.2 degrees, heart...   \n",
+       "2                            Yes, both are positive.   \n",
+       "3  Labs are in order. White blood cell count of 1...   \n",
+       "4  Are there any other therapeutic options for th...   \n",
+       "\n",
+       "                                              answer  \n",
+       "0  Okay, well, I would begin by evaluating the pa...  \n",
+       "1  Okay, does he have a positive psoas or a Rovsi...  \n",
+       "2  I'm concerned about acute appendicitis, but th...  \n",
+       "3  Well, this is diagnostic of acute appendicitis...  \n",
+       "4  Yes, the patient has early acute appendicitis ...  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataframe_btk = process_all_cases(DATA_FOLDER)\n",
+    "print(f\"Loaded {dataframe_btk['case_id'].nunique()} cases. Displaying head:\")\n",
+    "display(dataframe_btk.head())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "prepare-rag-markdown",
+   "metadata": {},
+   "source": [
+    "## 3. Prepare RAG Dataset with Embeddings\n",
+    "\n",
+    "Use the `ClinicalCaseProcessor` to process the DataFrame, generate embeddings for each case summary, and save the result as a Hugging Face Dataset."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "prepare-rag-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Initializing ClinicalCaseProcessor with model: all-MiniLM-L6-v2\n",
+      "Using device: cuda\n",
+      "Using provided DataFrame.\n",
+      "Raw data shape: (973, 5)\n",
+      "Grouping data by case...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Processing Cases: 100%|██████████| 99/99 [00:00<00:00, 6184.54it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed data into 99 unique cases.\n",
+      "Generating embeddings for 99 case summaries...\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Embedding Batches: 100%|██████████| 7/7 [00:00<00:00, 11.19it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generated embeddings with shape: (99, 384)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ef6d95d464ea4babb9adeb1ff9b16582",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Saving the dataset (0/1 shards):   0%|          | 0/99 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed dataset saved successfully to ../processed_clinical_cases\n",
+      "Dataset({\n",
+      "    features: ['case_id', 'clinical_presentation', 'questions', 'answers', 'case_summary', 'embeddings'],\n",
+      "    num_rows: 99\n",
+      "})\n"
+     ]
+    }
+   ],
+   "source": [
+    "processor = ClinicalCaseProcessor(model_name=EMBEDDING_MODEL_ID)\n",
+    "processed_dataset = processor.preprocess_data(dataframe_btk, output_path=PROCESSED_DATA_PATH)\n",
+    "print(processed_dataset)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "init-pipeline-markdown",
+   "metadata": {},
+   "source": [
+    "## 4. Initialize RAG Pipeline Components\n",
+    "\n",
+    "Instantiate the retriever, evaluator, and simulator classes."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "init-pipeline-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Initializing ClinicalCaseRetriever with model: all-MiniLM-L6-v2\n",
+      "Dataset loaded successfully from disk: ../processed_clinical_cases\n",
+      "Dataset features: {'case_id': Value(dtype='string', id=None), 'clinical_presentation': Value(dtype='string', id=None), 'questions': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), 'answers': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), 'case_summary': Value(dtype='string', id=None), 'embeddings': Sequence(feature=Value(dtype='float64', id=None), length=-1, id=None)}\n",
+      "Number of cases in dataset: 99\n",
+      "Using device: cuda\n",
+      "Loaded 99 cases with embeddings of shape (99, 384)\n",
+      "Initializing AnswerEvaluator with model: meta-llama/Llama-3.2-3B-Instruct\n",
+      "Set pad_token to eos_token\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "02a5e6af243344489487ed8639da9db3",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "AnswerEvaluator model loaded successfully on device: cuda:0\n"
+     ]
+    }
+   ],
+   "source": [
+    "retriever = ClinicalCaseRetriever(dataset_path=PROCESSED_DATA_PATH, model_name=EMBEDDING_MODEL_ID)\n",
+    "evaluator = AnswerEvaluator(model_id=EVALUATOR_MODEL_ID)\n",
+    "simulator = OralExamSimulator(retriever, evaluator)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "run-sim-markdown",
+   "metadata": {},
+   "source": [
+    "## 5. Run Simulation with RAG\n",
+    "\n",
+    "Demonstrate the core simulation loop: start a case based on a query, process user responses, and get feedback.  \n",
+    "Let's take the example of 'bowel intussusception in a child'."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "run-sim-start-markdown",
+   "metadata": {},
+   "source": [
+    "### 5.1 Start a New Case"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "run-sim-start-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "**➡️ Please enter the clinical scenario that you would like to be examined on:**"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "   Your Answer:  bowel intussusception in child\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--------------------------------------------------\n",
+      "Attempting to start new case | Query: 'bowel intussusception in child' | Index: None\n",
+      "Encoding query: 'Clinical case about bowel intussusception in child'\n",
+      "Retrieved 1 cases with similarity scores:\n",
+      "- Intussusception Pediatrics: 0.7762\n",
+      "Retrieved case via query ('bowel intussusception in child') with score 0.7762: Intussusception Pediatrics\n",
+      "Case successfully started. Total questions: 12\n",
+      "--------------------------------------------------\n",
+      "\n",
+      "Case Started: Intussusception Pediatrics\n",
+      "Total Questions: 12\n",
+      "\n",
+      "--- Question 1 --- \n",
+      "You're called to the emergency department to evaluate an 18-month-old boy with an eight-hour history of intermittent intense abdominal pain. His mother notes that the pain starts abruptly, and that the kid will draw his knees up to his chest during these painful episodes. Pain lasts for 30 minutes at a time and spontaneously resolves. He's had one episode of non-bloody emesis. He's not had diarrhea or bloody stools.\n"
+     ]
+    }
+   ],
+   "source": [
+    "# --- Start Simulation ---\n",
+    "display(Markdown(f\"**➡️ Please enter the clinical scenario that you would like to be examined on:**\"))\n",
+    "query = input(\"   Your Answer: \")\n",
+    "case_info = simulator.start_new_case(clinical_query=query)\n",
+    "\n",
+    "# Directly print the first question (assuming success)\n",
+    "print(f\"\\nCase Started: {case_info.get('clinical_presentation', 'N/A')}\")\n",
+    "print(f\"Total Questions: {case_info.get('total_questions', 'N/A')}\")\n",
+    "print(\"\\n--- Question 1 --- \")\n",
+    "print(case_info.get('current_question', 'Error: No question found'))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "run-sim-turn1-markdown",
+   "metadata": {},
+   "source": [
+    "### 5.2 Process User Response (Turn 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "44c6e822-5508-4db2-8700-79f25dc14e05",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "**➡️ Your Turn (Question 1/12)**"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "   Your Answer:  Perform physical exam, looking specifically for abdominal signs, order labs, and get vitals\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "⏳ Processing User Answer...\n",
+      "--------------------------------------------------\n",
+      "Processing response for Question 1/12\n",
+      "User Response: Perform physical exam, looking specifically for abdominal signs, order labs, and get vitals\n",
+      "Expected Answer: I'd begin by obtaining vital signs and performing a comprehensive history and physical examination, focusing on his history, birth history, and my exam focusing on his abdominal exam, looking for hernias, and doing a rectal exam as well.\n",
+      "Generated Feedback: The resident’s response is: \n",
+      "Perform physical exam, looking specifically for abdominal signs, order labs, and get vitals\n",
+      "\n",
+      "ASSESSMENT: Partially Correct\n",
+      "\n",
+      "The resident mentioned performing a physical exam and ordering labs, but missed the importance of getting vitals. The resident’s response is partially correct because they included some key points but omitted others, such as the importance of obtaining vital signs.\n",
+      "Next question (1/12): The patient's tachycardic, the rest of the vital signs are normal. Your exam reveals a toddler in the fetal position on the stretcher. He appears to be in moderate distress. He has diffuse abdominal tenderness and voluntary guarding.\n",
+      "--------------------------------------------------\n",
+      "\n",
+      "❓Next Question (1/12)\n",
+      "\n",
+      "The patient's tachycardic, the rest of the vital signs are normal. Your exam reveals a toddler in the fetal position on the stretcher. He appears to be in moderate distress. He has diffuse abdominal tenderness and voluntary guarding.\n"
+     ]
+    }
+   ],
+   "source": [
+    "display(Markdown(f\"**➡️ Your Turn (Question {simulator.current_question_idx + 1}/{len(simulator.current_case['questions'])})**\"))\n",
+    "user_answer = input(\"   Your Answer: \")\n",
+    "\n",
+    "# --- Process Response ---\n",
+    "print(f\"\\n⏳ Processing User Answer...\")\n",
+    "result = simulator.process_user_response(user_answer)\n",
+    "\n",
+    "# Next question\n",
+    "q_num = result.get('question_number', '?')\n",
+    "next_q_text = result.get('next_question', '*Error: No next question found*')\n",
+    "print(f\"\\n❓Next Question ({q_num}/{len(simulator.current_case['questions'])})\\n\\n{next_q_text}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "run-sim-turn2-markdown",
+   "metadata": {},
+   "source": [
+    "### 5.3 Process User Response (Turn 2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "run-sim-turn2-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "**➡️ Your Turn (Question 2/12)**"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "   Your Answer:  Ask for labs, get imaging\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "⏳ Processing User Answer...\n",
+      "--------------------------------------------------\n",
+      "Processing response for Question 2/12\n",
+      "User Response: Ask for labs, get imaging\n",
+      "Expected Answer: I'd like to know a little bit more about the history. Has he recently been ill or had any sick contacts? When did he pass stool or gas last? Is this a first such episode? Is there any family history of GI conditions or malignancy?\n",
+      "Generated Feedback: The resident’s response is: \n",
+      "I'd like to know a little bit more about the patient's history. Has he recently been ill or had any sick contacts? When did he pass stool or gas last? Is this a first episode? Is there any family history of GI conditions or malignancy? Ask for labs, get imaging.\n",
+      "\n",
+      "The expected answer is: \n",
+      "I'd like to know a little bit more about the patient's history. Has he recently been ill or had any sick contacts? When did he pass stool or gas last? Is this a first episode? Is there any family history of GI conditions or malignancy? Ask for labs, get imaging.\n",
+      "\n",
+      "The resident’s response is identical to the expected answer. The resident includes all the\n",
+      "Next question (2/12): He has had a mild upper respiratory infection for the past week, but he's been eating normally. His last bowel movement was yesterday. This is the first time he's ever had symptoms like this, and his family and personal history is otherwise unremarkable.\n",
+      "--------------------------------------------------\n",
+      "\n",
+      "❓ Next Question (2/12)\n",
+      "\n",
+      "He has had a mild upper respiratory infection for the past week, but he's been eating normally. His last bowel movement was yesterday. This is the first time he's ever had symptoms like this, and his family and personal history is otherwise unremarkable.\n"
+     ]
+    }
+   ],
+   "source": [
+    "display(Markdown(f\"**➡️ Your Turn (Question {simulator.current_question_idx + 1}/{len(simulator.current_case['questions'])})**\"))\n",
+    "user_answer = input(\"   Your Answer: \")\n",
+    "\n",
+    "print(f\"\\n⏳ Processing User Answer...\")\n",
+    "result = simulator.process_user_response(user_answer)\n",
+    "\n",
+    "q_num = result.get('question_number', '?')\n",
+    "next_q_text = result.get('next_question', '*Error: No next question found*')\n",
+    "print(f\"\\n❓ Next Question ({q_num}/{len(simulator.current_case['questions'])})\\n\\n{next_q_text}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "run-sim-summary-markdown",
+   "metadata": {},
+   "source": [
+    "### 5.4 Session Summary and Saving"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "run-sim-summary-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "--- Generating Session Summary --- \n",
+      "Case: Intussusception Pediatrics\n",
+      "Case ID: 87A\n",
+      "Total Questions in Case: 12\n",
+      "Number of Interactions Logged: 8\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"\\n--- Generating Session Summary --- \")\n",
+    "session_summary = simulator.generate_session_summary()\n",
+    "\n",
+    "print(f\"Case: {session_summary.get('case', 'N/A')}\")\n",
+    "print(f\"Case ID: {session_summary.get('case_id', 'N/A')}\")\n",
+    "print(f\"Total Questions in Case: {session_summary.get('total_questions_in_case', 'N/A')}\")\n",
+    "print(f\"Number of Interactions Logged: {len(session_summary.get('interaction_history', []))}\")\n",
+    "\n",
+    "# Display full session interaction\n",
+    "#print(json.dumps(session_summary, indent=2)) # Keep commented out or remove"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eval-retrieval-markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Evaluate Retrieval Performance\n",
+    "\n",
+    "Creating dummy dataset to evaluate RAG performance"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "eval-retrieval-run-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Calculating retrieval metrics for 5 queries (k=5)...\n",
+      "\n",
+      "Processing query 1/5: 'appendix inflammation in a child' (Expected ID: '85A')\n",
+      "Encoding query: 'Clinical case about appendix inflammation in a child'\n",
+      "Retrieved 5 cases with similarity scores:\n",
+      "- Appendicitis Pediatrics: 0.7076\n",
+      "- Acute Appendicitis: 0.5978\n",
+      "- Intussusception Pediatrics: 0.5745\n",
+      "- Meckel s Diverticulum Pediatrics: 0.5277\n",
+      "- Abdominal Mass Pediatrics: 0.4705\n",
+      "Retrieved IDs: ['85A', '31A', '87A', '89A', '84A']\n",
+      "Retrieved Scores: [0.7076, 0.5978, 0.5745, 0.5277, 0.4705]\n",
+      "Hit: 1, Rank: 1, NDCG@5: 1.0000\n",
+      "\n",
+      "Processing query 2/5: 'Pyloric Stenosis in a child' (Expected ID: '90A')\n",
+      "Encoding query: 'Clinical case about Pyloric Stenosis in a child'\n",
+      "Retrieved 5 cases with similarity scores:\n",
+      "- Pyloric Stenosis Pediatrics: 0.7554\n",
+      "- Carotid Stenosis: 0.4458\n",
+      "- Pheochromocytoma: 0.4139\n",
+      "- Small Bowel Obstruction: 0.3876\n",
+      "- Intussusception Pediatrics: 0.3855\n",
+      "Retrieved IDs: ['90A', '79A', '54A', '23A', '87A']\n",
+      "Retrieved Scores: [0.7554, 0.4458, 0.4139, 0.3876, 0.3855]\n",
+      "Hit: 1, Rank: 1, NDCG@5: 1.0000\n",
+      "\n",
+      "Processing query 3/5: 'perforation of the esophagus' (Expected ID: '16A')\n",
+      "Encoding query: 'Clinical case about perforation of the esophagus'\n",
+      "Retrieved 5 cases with similarity scores:\n",
+      "- Esophageal Perforation: 0.6348\n",
+      "- Esophageal Cancer: 0.5415\n",
+      "- Esophageal Dysmotility: 0.5269\n",
+      "- ARDS: 0.4708\n",
+      "- Esophagus and Trachea Trauma Empyema: 0.4500\n",
+      "Retrieved IDs: ['16A', '14A', '15A', '65A', '71A']\n",
+      "Retrieved Scores: [0.6348, 0.5415, 0.5269, 0.4708, 0.45]\n",
+      "Hit: 1, Rank: 1, NDCG@5: 1.0000\n",
+      "\n",
+      "Processing query 4/5: 'papilloma of the breast' (Expected ID: '43A')\n",
+      "Encoding query: 'Clinical case about papilloma of the breast'\n",
+      "Retrieved 5 cases with similarity scores:\n",
+      "- Intraductal Papilloma: 0.7125\n",
+      "- Unknown: 0.4971\n",
+      "- Inflammatory Breast Cancer: 0.4842\n",
+      "- Breast Abscess: 0.4826\n",
+      "- Breast Cancer in Pregnancy: 0.4616\n",
+      "Retrieved IDs: ['43A', 'Unknown', '42A', '39A', '40A']\n",
+      "Retrieved Scores: [0.7125, 0.4971, 0.4842, 0.4826, 0.4616]\n",
+      "Hit: 1, Rank: 1, NDCG@5: 1.0000\n",
+      "\n",
+      "Processing query 5/5: 'injury to the neck vessel' (Expected ID: '66A')\n",
+      "Encoding query: 'Clinical case about injury to the neck vessel'\n",
+      "Retrieved 5 cases with similarity scores:\n",
+      "- Vascular Neck Injury: 0.6604\n",
+      "- Esophagus and Trachea Trauma Empyema: 0.4970\n",
+      "- Liver Trauma: 0.4957\n",
+      "- Carotid Stenosis: 0.4798\n",
+      "- Lower Extremity Vascular Injury: 0.4598\n",
+      "Retrieved IDs: ['66A', '71A', '72A', '79A', '68A']\n",
+      "Retrieved Scores: [0.6604, 0.497, 0.4957, 0.4798, 0.4598]\n",
+      "Hit: 1, Rank: 1, NDCG@5: 1.0000\n",
+      "\n",
+      "--- Overall Retrieval Results (k=5) --- \n",
+      "Average Hit@5: 1.0000\n",
+      "Average MRR:    1.0000\n",
+      "Average NDCG@5: 1.0000\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define benchmark queries and corresponding gold standard case IDs\n",
+    "benchmark_queries = [\n",
+    "    \"appendix inflammation in a child\",\n",
+    "    \"Pyloric Stenosis in a child\",\n",
+    "    \"perforation of the esophagus\",\n",
+    "    \"papilloma of the breast\",\n",
+    "    \"injury to the neck vessel\"\n",
+    "]\n",
+    "\n",
+    "benchmark_gold_ids = [\n",
+    "    \"85A\", # Appendicitis Pediatrics\n",
+    "    \"90A\", # Pyloric Stenosis Pediatrics\n",
+    "    \"16A\", # Esophageal Perforation\n",
+    "    \"43A\", # Intraductal Papilloma\n",
+    "    \"66A\"  # Vascular Neck Injury\n",
+    "]\n",
+    "\n",
+    "# Run benchmarking at k=5 using the initialized retriever\n",
+    "k_value = 5\n",
+    "results = retrieval_metrics(retriever, benchmark_queries, benchmark_gold_ids, k=k_value)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "compare-nonrag-markdown",
+   "metadata": {},
+   "source": [
+    "## 7. Compare with Non-RAG Simulation\n",
+    "\n",
+    "This section demonstrates generating a synthetic case directly with an LLM and running the simulation using the `DummyRetriever` for comparison.  \n",
+    "We will use the same example (intussusception in pediatric patient')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "compare-nonrag-gen-code",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/markdown": [
+       "**➡️ Please enter the topic that you would like to generate synthetic data about:**"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdin",
+     "output_type": "stream",
+     "text": [
+      "Answer:  bowel intussusception child\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Generating synthetic case for 'bowel intussusception child' using meta-llama/Llama-3.2-3B-Instruct...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c7501d4d2b714cb7b5c8bc7199138a85",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Synthetic case generation complete.\n",
+      "Processed synthetic data into DataFrame with 8 turns.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>case_id</th>\n",
+       "      <th>clinical_presentation</th>\n",
+       "      <th>turn_id</th>\n",
+       "      <th>question</th>\n",
+       "      <th>answer</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>1</td>\n",
+       "      <td>**\\nA 4-year-old boy presents to the emergency...</td>\n",
+       "      <td>Intussusception\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>2</td>\n",
+       "      <td>**\\nWhat is the typical age range for intussus...</td>\n",
+       "      <td>6-36 months\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>3</td>\n",
+       "      <td>**\\nWhat is the primary mechanism of intussusc...</td>\n",
+       "      <td>Invagination of a lead point, often a tumor or...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>4</td>\n",
+       "      <td>**\\nWhich of the following is a common finding...</td>\n",
+       "      <td>A palpable abdominal mass\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>5</td>\n",
+       "      <td>**\\nWhat is the most common location for intus...</td>\n",
+       "      <td>Ileum\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>6</td>\n",
+       "      <td>**\\nWhat is the primary treatment for intussus...</td>\n",
+       "      <td>Reduction with air enema\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>7</td>\n",
+       "      <td>**\\nWhat is a potential complication of intuss...</td>\n",
+       "      <td>Perforation and peritonitis\\n\\n**</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>SYNTH_01</td>\n",
+       "      <td>bowel intussusception child</td>\n",
+       "      <td>8</td>\n",
+       "      <td>**\\nHow often does intussusception recur after...</td>\n",
+       "      <td>Less than 5% of cases, with most recurrences o...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "    case_id        clinical_presentation  turn_id  \\\n",
+       "0  SYNTH_01  bowel intussusception child        1   \n",
+       "1  SYNTH_01  bowel intussusception child        2   \n",
+       "2  SYNTH_01  bowel intussusception child        3   \n",
+       "3  SYNTH_01  bowel intussusception child        4   \n",
+       "4  SYNTH_01  bowel intussusception child        5   \n",
+       "5  SYNTH_01  bowel intussusception child        6   \n",
+       "6  SYNTH_01  bowel intussusception child        7   \n",
+       "7  SYNTH_01  bowel intussusception child        8   \n",
+       "\n",
+       "                                            question  \\\n",
+       "0  **\\nA 4-year-old boy presents to the emergency...   \n",
+       "1  **\\nWhat is the typical age range for intussus...   \n",
+       "2  **\\nWhat is the primary mechanism of intussusc...   \n",
+       "3  **\\nWhich of the following is a common finding...   \n",
+       "4  **\\nWhat is the most common location for intus...   \n",
+       "5  **\\nWhat is the primary treatment for intussus...   \n",
+       "6  **\\nWhat is a potential complication of intuss...   \n",
+       "7  **\\nHow often does intussusception recur after...   \n",
+       "\n",
+       "                                              answer  \n",
+       "0                              Intussusception\\n\\n**  \n",
+       "1                                  6-36 months\\n\\n**  \n",
+       "2  Invagination of a lead point, often a tumor or...  \n",
+       "3                    A palpable abdominal mass\\n\\n**  \n",
+       "4                                        Ileum\\n\\n**  \n",
+       "5                     Reduction with air enema\\n\\n**  \n",
+       "6                  Perforation and peritonitis\\n\\n**  \n",
+       "7  Less than 5% of cases, with most recurrences o...  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# --- Generate and Process Synthetic Case ---\n",
+    "display(Markdown(f\"**➡️ Please enter the topic that you would like to generate synthetic data about:**\"))\n",
+    "synthetic_query = input(\"Answer: \")\n",
+    "df_synthetic_case = pd.DataFrame()\n",
+    "\n",
+    "raw_synthetic_text = generate_synthetic_case(synthetic_query, model_id=EVALUATOR_MODEL_ID)\n",
+    "df_synthetic_case = process_synthetic_data(synthetic_query, raw_synthetic_text)\n",
+    "display(df_synthetic_case)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9698b316-528f-4324-a6a5-ca27b1407b12",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "DummyRetriever processing 1 unique presentations.\n",
+      "DummyRetriever initialized with 1 cases.\n",
+      "--------------------------------------------------\n",
+      "Attempting to start new case | Query: 'bowel intussusception child' | Index: None\n",
+      "DummyRetriever searching for exact match: 'bowel intussusception child'\n",
+      "DummyRetriever found match: bowel intussusception child\n",
+      "Retrieved case via query ('bowel intussusception child') with score 1.0000: bowel intussusception child\n",
+      "Case successfully started. Total questions: 8\n",
+      "--------------------------------------------------\n",
+      "[Q1] **\n",
+      "A 4-year-old boy presents to the emergency department with abdominal pain and vomiting. The child was playing outside and suddenly became ill, unable to continue playing. He has no history of abdominal pain or gastrointestinal symptoms.\n",
+      "\n",
+      "**Q1:**\n",
+      "What is the likely underlying cause of the child's symptoms?\n",
+      "A1: Intussusception\n",
+      "\n",
+      "**Q2:**\n",
+      "What is the typical age range for intussusception in children?\n",
+      "A2: 6-36 months\n",
+      "\n",
+      "**Q3:**\n",
+      "What is the primary mechanism of intussusception in children?\n",
+      "A3: Invagination of a lead point, often a tumor or inflammatory lesion, into the intestine\n",
+      "\n",
+      "**Q4:**\n",
+      "Which of the following is a common finding in children with intussusception?\n",
+      "A4: A palpable abdominal mass\n",
+      "\n",
+      "**Q5:**\n",
+      "What is the most common location for intussusception in children?\n",
+      "A5: Ileum\n",
+      "\n",
+      "**Q6:**\n",
+      "What is the primary treatment for intussusception in children?\n",
+      "A6: Reduction with air enema\n",
+      "\n",
+      "**Q7:**\n",
+      "What is a potential complication of intussusception if not treated promptly?\n",
+      "A7: Perforation and peritonitis\n",
+      "\n",
+      "**Q8:**\n",
+      "How often does intussusception recur after successful treatment?\n",
+      "A8: Less than 5% of cases, with most recurrences occurring within 6 months of initial presentation. **\n",
+      "What is the likely underlying cause of the child's symptoms?\n"
+     ]
+    },
+    {
+     "data": {
+      "text/markdown": [
+       "<br><b>Please enter your answer:"
+      ],
+      "text/plain": [
+       "<IPython.core.display.Markdown object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Initialize components and starting case\n",
+    "dummy_retriever = DummyRetriever(df_synthetic_case)\n",
+    "non_rag_simulator = OralExamSimulator(dummy_retriever, evaluator)\n",
+    "non_rag_case_info = non_rag_simulator.start_new_case(clinical_query=synthetic_query)\n",
+    "print(f\"[Q1] {non_rag_case_info.get('current_question', 'N/A')}\")\n",
+    "\n",
+    "display(Markdown(f\"<br><b>Please enter your answer:\"))\n",
+    "non_rag_answer_1 = input(\"Answer :\")\n",
+    "non_rag_result_1 = non_rag_simulator.process_user_response(non_rag_answer_1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f5ec35c2-1bc0-4602-a194-52fa62acb94d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# --- Process Turn 2 ---\n",
+    "print(f\"[Q2] {non_rag_result_1.get('next_question', 'N/A')}\")\n",
+    "display(Markdown(f\"<br><b>Please enter your answer:\"))\n",
+    "non_rag_answer_2 = input(\"Answer :\")\n",
+    "non_rag_answer_2 = non_rag_simulator.process_user_response(non_rag_answer_2)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "llm_course",
+   "language": "python",
+   "name": "llm_course"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}