Upload 43 files

.gitattributes

@@ -33,3 +33,28 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_storage/faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20appendix%204.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%201.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2010.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2011.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2012.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2013.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2014.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2015.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2016.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2017.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2018.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2019.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%202.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2020.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2021.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%2022.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%203.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%204.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%205.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%206.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%207.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%208.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics%20cap%209.pdf filter=lfs diff=lfs merge=lfs -text
+sources/preguntas[[:space:]]chatbot_01.xlsx filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,35 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1-mesa-glx \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file
+COPY requirements.txt requirements.txt
+
+# Install Python packages
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . /app
+
+# Create a non-root user
+RUN useradd -m -u 1000 user
+
+# Change ownership
+RUN chown -R user:user /app
+
+# Switch to the non-root user
+USER user
+
+# Expose the port Gunicorn will run on (Using 7860 as in CMD)
+EXPOSE 7860
+
+# Command to run the app
+CMD ["python", "app.py", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -1,64 +1,1037 @@
-import gradio as gr
-from huggingface_hub import InferenceClient
-
-"""
-For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-"""
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-
-
-def respond(
-    message,
-    history: list[tuple[str, str]],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [{"role": "system", "content": system_message}]
-
-    for val in history:
-        if val[0]:
-            messages.append({"role": "user", "content": val[0]})
-        if val[1]:
-            messages.append({"role": "assistant", "content": val[1]})
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
-
-        response += token
-        yield response
-
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-demo = gr.ChatInterface(
-    respond,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
-
-
-if __name__ == "__main__":
-    demo.launch()
+from flask import Flask, request, send_file, abort, jsonify, url_for, render_template, Response
+from flask_cors import CORS
+import pandas as pd
+from sentence_transformers import SentenceTransformer, util
+import torch
+from dataclasses import dataclass
+from typing import List, Dict, Tuple, Optional, Any, Iterator
+from collections import deque
+import os
+import logging
+import atexit
+from threading import Thread, Lock
+import time
+from datetime import datetime
+from uuid import uuid4 as generate_uuid
+import csv as csv_lib
+import functools
+import json
+import re
+import subprocess
+import sys
+import sqlite3
+import io
+
+from dotenv import load_dotenv
+
+# Load environment variables from .env file AT THE VERY TOP
+load_dotenv()
+
+# Import RAG system and Fallback LLM from llm_handling AFTER load_dotenv
+# MODIFIED: Imported new functions and prompts
+from llm_handling import (
+    initialize_and_get_rag_system,
+    KnowledgeRAG,
+    groq_bot_instance,
+    RAG_SOURCES_DIR,
+    RAG_STORAGE_PARENT_DIR,
+    RAG_CHUNKED_SOURCES_FILENAME,
+    get_answer_from_context
+)
+from system_prompts import QA_FORMATTER_PROMPT
+
+
+# Setup logging (remains global for the app)
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler("app_hybrid_rag.log"),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__) # Main app logger
+
+# --- Application Constants and Configuration ---
+ADMIN_USERNAME = os.getenv('FLASK_ADMIN_USERNAME', 'admin')
+ADMIN_PASSWORD = os.getenv('FLASK_ADMIN_PASSWORD', 'admin')
+FLASK_APP_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_APP_PORT = int(os.getenv("FLASK_PORT", "7860"))
+FLASK_DEBUG_MODE = os.getenv("FLASK_DEBUG", "True").lower() == "true"
+_APP_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEXT_EXTRACTIONS_DIR = os.path.join(_APP_BASE_DIR, 'text_extractions')
+RELATED_QUESTIONS_TO_SHOW = 10
+QUESTIONS_TO_SEND_TO_GROQ_QA = 3
+# MODIFIED: Replaced separate confidence values with a single configurable one for the LLM formatter.
+LLM_FORMATTER_CONFIDENCE_THRESHOLD = int(os.getenv("LLM_FORMATTER_CONFIDENCE_THRESHOLD", "95"))
+HIGH_CONFIDENCE_THRESHOLD = 90 # For greetings, which are answered directly without LLM formatting.
+# MODIFIED: Made CHAT_HISTORY_TO_SEND configurable via environment variable
+CHAT_HISTORY_TO_SEND = int(os.getenv("CHAT_HISTORY_TO_SEND", "5")) # Defines how many *pairs* of (user, assistant) messages to send
+CHAT_LOG_FILE = os.path.join(_APP_BASE_DIR, 'chat_history.csv')
+
+rag_system: Optional[KnowledgeRAG] = None
+
+# --- Persistent Chat History Management using SQLite ---
+class ChatHistoryManager:
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.lock = Lock()
+        self._create_table()
+        logger.info(f"SQLite chat history manager initialized at: {self.db_path}")
+
+    def _get_connection(self):
+        conn = sqlite3.connect(self.db_path, timeout=10)
+        return conn
+
+    def _create_table(self):
+        with self.lock:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS chat_histories (
+                        session_id TEXT PRIMARY KEY,
+                        history TEXT NOT NULL
+                    )
+                """)
+                conn.commit()
+
+    def get_history(self, session_id: str, limit_turns: int = 5) -> list:
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT history FROM chat_histories WHERE session_id = ?", (session_id,))
+                row = cursor.fetchone()
+                if row:
+                    history_list = json.loads(row[0])
+                    return history_list[-(limit_turns * 2):]
+                else:
+                    return []
+        except Exception as e:
+            logger.error(f"Error fetching history for session {session_id}: {e}", exc_info=True)
+            return []
+
+    def update_history(self, session_id: str, query: str, answer: str):
+        with self.lock:
+            try:
+                with self._get_connection() as conn:
+                    cursor = conn.cursor()
+                    cursor.execute("SELECT history FROM chat_histories WHERE session_id = ?", (session_id,))
+                    row = cursor.fetchone()
+                    
+                    history = json.loads(row[0]) if row else []
+                    
+                    history.append({'role': 'user', 'content': query})
+                    history.append({'role': 'assistant', 'content': answer})
+
+                    updated_history_json = json.dumps(history)
+
+                    cursor.execute("""
+                        INSERT OR REPLACE INTO chat_histories (session_id, history)
+                        VALUES (?, ?)
+                    """, (session_id, updated_history_json))
+                    conn.commit()
+            except Exception as e:
+                logger.error(f"Error updating history for session {session_id}: {e}", exc_info=True)
+                
+    def clear_history(self, session_id: str):
+        with self.lock:
+            try:
+                with self._get_connection() as conn:
+                    cursor = conn.cursor()
+                    cursor.execute("""
+                        INSERT OR REPLACE INTO chat_histories (session_id, history)
+                        VALUES (?, ?)
+                    """, (session_id, json.dumps([])))
+                    conn.commit()
+                logger.info(f"Chat history cleared for session: {session_id}")
+            except Exception as e:
+                 logger.error(f"Error clearing history for session {session_id}: {e}", exc_info=True)
+
+
+# --- EmbeddingManager for CSV QA (remains in app.py) ---
+@dataclass
+class QAEmbeddings:
+    questions: List[str]
+    question_map: List[int]
+    embeddings: torch.Tensor
+    df_qa: pd.DataFrame
+    original_questions: List[str]
+
+class EmbeddingManager:
+    def __init__(self, model_name='all-MiniLM-L6-v2'):
+        self.model = SentenceTransformer(model_name)
+        self.embeddings = {
+            'general': None,
+            'personal': None,
+            'greetings': None
+        }
+        logger.info(f"EmbeddingManager initialized with model: {model_name}")
+
+    def _process_questions(self, df: pd.DataFrame) -> Tuple[List[str], List[int], List[str]]:
+        questions = []
+        question_map = []
+        original_questions = []
+
+        if 'Question' not in df.columns:
+            logger.warning(f"DataFrame for EmbeddingManager is missing 'Question' column. Cannot process questions from it.")
+            return questions, question_map, original_questions
+
+        for idx, question_text_raw in enumerate(df['Question']):
+            if pd.isna(question_text_raw):
+                continue
+            question_text_cleaned = str(question_text_raw).strip()
+            if not question_text_cleaned or question_text_cleaned.lower() == "nan":
+                continue
+
+            questions.append(question_text_cleaned)
+            question_map.append(idx)
+            original_questions.append(question_text_cleaned)
+
+        return questions, question_map, original_questions
+
+    def update_embeddings(self, general_qa: pd.DataFrame, personal_qa: pd.DataFrame, greetings_qa: pd.DataFrame):
+        gen_questions, gen_question_map, gen_original_questions = self._process_questions(general_qa)
+        gen_embeddings = self.model.encode(gen_questions, convert_to_tensor=True, show_progress_bar=False) if gen_questions else None
+
+        pers_questions, pers_question_map, pers_original_questions = self._process_questions(personal_qa)
+        pers_embeddings = self.model.encode(pers_questions, convert_to_tensor=True, show_progress_bar=False) if pers_questions else None
+
+        greet_questions, greet_question_map, greet_original_questions = self._process_questions(greetings_qa)
+        greet_embeddings = self.model.encode(greet_questions, convert_to_tensor=True, show_progress_bar=False) if greet_questions else None
+
+        self.embeddings['general'] = QAEmbeddings(
+            questions=gen_questions, question_map=gen_question_map, embeddings=gen_embeddings,
+            df_qa=general_qa, original_questions=gen_original_questions
+        )
+        self.embeddings['personal'] = QAEmbeddings(
+            questions=pers_questions, question_map=pers_question_map, embeddings=pers_embeddings,
+            df_qa=personal_qa, original_questions=pers_original_questions
+        )
+        self.embeddings['greetings'] = QAEmbeddings(
+            questions=greet_questions, question_map=greet_question_map, embeddings=greet_embeddings,
+            df_qa=greetings_qa, original_questions=greet_original_questions
+        )
+        logger.info("CSV QA embeddings updated in EmbeddingManager.")
+
+    def find_best_answers(self, user_query: str, qa_type: str, top_n: int = 5) -> Tuple[List[float], List[str], List[str], List[str], List[int]]:
+        qa_data = self.embeddings[qa_type]
+        if qa_data is None or qa_data.embeddings is None or len(qa_data.embeddings) == 0:
+            return [], [], [], [], []
+
+        query_embedding_tensor = self.model.encode([user_query], convert_to_tensor=True, show_progress_bar=False)
+        if not isinstance(qa_data.embeddings, torch.Tensor):
+             qa_data.embeddings = torch.tensor(qa_data.embeddings) # Safeguard
+
+        cos_scores = util.cos_sim(query_embedding_tensor, qa_data.embeddings)[0]
+
+        top_k = min(top_n, len(cos_scores))
+        if top_k == 0:
+            return [], [], [], [], []
+
+        top_scores_tensor, indices_tensor = torch.topk(cos_scores, k=top_k)
+
+        top_confidences = [score.item() * 100 for score in top_scores_tensor]
+        top_indices_mapped = []
+        top_questions = []
+
+        for idx_tensor in indices_tensor:
+            item_idx = idx_tensor.item()
+            if item_idx < len(qa_data.question_map) and item_idx < len(qa_data.original_questions):
+                 original_df_idx = qa_data.question_map[item_idx]
+                 if original_df_idx < len(qa_data.df_qa):
+                    top_indices_mapped.append(original_df_idx)
+                    top_questions.append(qa_data.original_questions[item_idx])
+                 else:
+                    logger.warning(f"Index out of bounds: original_df_idx {original_df_idx} for df_qa length {len(qa_data.df_qa)}")
+            else:
+                logger.warning(f"Index out of bounds: item_idx {item_idx} for question_map/original_questions")
+
+        valid_count = len(top_indices_mapped)
+        top_confidences = top_confidences[:valid_count]
+        top_questions = top_questions[:valid_count]
+        
+        # MODIFIED: Changed Answer to Respuesta to match new loading logic for xlsx
+        answer_col = 'Respuesta' if 'Respuesta' in qa_data.df_qa.columns else 'Answer'
+        top_answers = [str(qa_data.df_qa[answer_col].iloc[i]) for i in top_indices_mapped]
+        top_images = [str(qa_data.df_qa['Image'].iloc[i]) if 'Image' in qa_data.df_qa.columns and pd.notna(qa_data.df_qa['Image'].iloc[i]) else None for i in top_indices_mapped]
+
+        return top_confidences, top_questions, top_answers, top_images, top_indices_mapped
+
+# --- DatabaseMonitor for personal_qa.csv placeholders (remains in app.py) ---
+class DatabaseMonitor:
+    def __init__(self, database_path):
+        self.logger = logging.getLogger(__name__ + ".DatabaseMonitor")
+        self.database_path = database_path
+        self.last_modified = None
+        self.last_size = None
+        self.df = None
+        self.lock = Lock()
+        self.running = True
+        self._load_database()
+        self.monitor_thread = Thread(target=self._monitor_database, daemon=True)
+        self.monitor_thread.start()
+        self.logger.info(f"DatabaseMonitor initialized for: {database_path}")
+
+    def _load_database(self):
+        try:
+            if not os.path.exists(self.database_path):
+                self.logger.warning(f"Personal data file not found: {self.database_path}.")
+                self.df = None
+                return
+            with self.lock:
+                self.df = pd.read_csv(self.database_path, encoding='cp1252')
+                self.last_modified = os.path.getmtime(self.database_path)
+                self.last_size = os.path.getsize(self.database_path)
+                self.logger.info(f"Personal data file reloaded: {self.database_path}")
+        except Exception as e:
+            self.logger.error(f"Error loading personal data file '{self.database_path}': {e}", exc_info=True)
+            self.df = None
+
+    def _monitor_database(self):
+        while self.running:
+            try:
+                if not os.path.exists(self.database_path):
+                    if self.df is not None:
+                        self.logger.warning(f"Personal data file disappeared: {self.database_path}")
+                        self.df = None; self.last_modified = None; self.last_size = None
+                    time.sleep(5)
+                    continue
+                current_modified = os.path.getmtime(self.database_path); current_size = os.path.getsize(self.database_path)
+                if (self.last_modified is None or current_modified != self.last_modified or
+                    self.last_size is None or current_size != self.last_size):
+                    self.logger.info("Personal data file change detected.")
+                    self._load_database()
+                time.sleep(1)
+            except Exception as e:
+                self.logger.error(f"Error monitoring personal data file: {e}", exc_info=True)
+                time.sleep(5)
+
+    def get_data(self, user_id):
+        with self.lock:
+            if self.df is not None and user_id:
+                try:
+                    if 'id' not in self.df.columns:
+                        self.logger.warning("'id' column not found in personal_data.csv")
+                        return None
+                    id_col_type = self.df['id'].dtype
+                    target_user_id = user_id
+                    if pd.api.types.is_numeric_dtype(id_col_type):
+                        try:
+                            if user_id is None: return None
+                            valid_ids = self.df['id'].dropna()
+                            if not valid_ids.empty:
+                                target_user_id = type(valid_ids.iloc[0])(user_id)
+                            else:
+                                target_user_id = int(user_id)
+                        except (ValueError, TypeError):
+                            self.logger.warning(f"Could not convert user_id '{user_id}' to numeric type {id_col_type}")
+                            return None
+                    user_data = self.df[self.df['id'] == target_user_id]
+                    if not user_data.empty: return user_data.iloc[0].to_dict()
+                except Exception as e:
+                    self.logger.error(f"Error retrieving data for user_id {user_id}: {e}", exc_info=True)
+            return None
+
+    def stop(self):
+        self.running = False
+        if hasattr(self, 'monitor_thread') and self.monitor_thread.is_alive():
+            self.monitor_thread.join(timeout=5)
+        self.logger.info("DatabaseMonitor stopped.")
+
+# --- Flask App Initialization ---
+app = Flask(__name__)
+CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True)
+
+# --- Initialize Managers ---
+embedding_manager = EmbeddingManager()
+history_manager = ChatHistoryManager('chat_history.db')
+database_csv_path = os.path.join(RAG_SOURCES_DIR, 'database.csv')
+personal_data_monitor = DatabaseMonitor(database_csv_path)
+
+# --- Helper Functions (App specific) ---
+def clean_html_from_text(text: str) -> str:
+    """Removes HTML tags from a string using a simple regex."""
+    if not isinstance(text, str):
+        return text
+    clean_text = re.sub(r'<[^>]+>', '', text)
+    return clean_text.strip()
+
+def normalize_text(text):
+    if isinstance(text, str):
+        replacements = {
+            '\x91': "'", '\x92': "'", '\x93': '"', '\x94': '"',
+            '\x96': '-', '\x97': '-', '\x85': '...', '\x95': '-',
+            '"': '"', '"': '"', '‘': "'", '’': "'",
+            '–': '-', '—': '-', '…': '...', '•': '-',
+        }
+        for old, new in replacements.items(): text = text.replace(old, new)
+    return text
+
+def require_admin_auth(f):
+    @functools.wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth or auth.username != ADMIN_USERNAME or auth.password != ADMIN_PASSWORD:
+            return Response('Admin auth failed.', 401, {'WWW-Authenticate': 'Basic realm="Admin Login Required"'})
+        return f(*args, **kwargs)
+    return decorated
+
+def initialize_chat_log():
+    if not os.path.exists(CHAT_LOG_FILE):
+        with open(CHAT_LOG_FILE, 'w', newline='', encoding='utf-8') as f:
+            writer = csv_lib.writer(f)
+            writer.writerow(['sl', 'date_time', 'session_id', 'user_id', 'query', 'answer'])
+
+def store_chat_history(sid: str, uid: Optional[str], query: str, resp: Dict[str, Any]):
+    try:
+        # This now gets the final response key, which is 'answer' in the old logic
+        answer = str(resp.get('answer', ''))
+        history_manager.update_history(sid, query, answer)
+
+        initialize_chat_log()
+        next_sl = 1
+        try:
+            if os.path.exists(CHAT_LOG_FILE) and os.path.getsize(CHAT_LOG_FILE) > 0:
+                df_log = pd.read_csv(CHAT_LOG_FILE, on_bad_lines='skip')
+                if not df_log.empty and 'sl' in df_log.columns and pd.api.types.is_numeric_dtype(df_log['sl'].dropna()):
+                    if not df_log['sl'].dropna().empty:
+                        next_sl = int(df_log['sl'].dropna().max()) + 1
+        except Exception as e:
+            logger.error(f"Error reading SL from {CHAT_LOG_FILE}: {e}", exc_info=True)
+
+        with open(CHAT_LOG_FILE, 'a', newline='', encoding='utf-8') as f:
+            csv_lib.writer(f).writerow([next_sl, datetime.now().strftime('%Y-%m-%d %H:%M:%S'), sid, uid or "N/A", query, answer])
+
+    except Exception as e:
+        logger.error(f"Error in store_chat_history for session {sid}: {e}", exc_info=True)
+
+def get_formatted_chat_history(session_id: str) -> List[Dict[str, str]]:
+    if not session_id:
+        return []
+    return history_manager.get_history(session_id, limit_turns=CHAT_HISTORY_TO_SEND)
+
+def get_qa_context_for_groq(all_questions: List[Dict]) -> str:
+    valid_qa_pairs = []
+    non_greeting_questions = [q for q in all_questions if q.get('source_type') != 'greetings']
+    sorted_questions = sorted(non_greeting_questions, key=lambda x: x.get('confidence', 0), reverse=True)
+
+    for qa in sorted_questions[:QUESTIONS_TO_SEND_TO_GROQ_QA]:
+        answer = qa.get('answer')
+        if (not pd.isna(answer) and isinstance(answer, str) and answer.strip() and
+            "not available" not in answer.lower()):
+            valid_qa_pairs.append(f"Q: {qa.get('question')}\nA: {answer}")
+    return '\n'.join(valid_qa_pairs)
+
+def replace_placeholders_in_answer(answer, db_data):
+    if pd.isna(answer) or str(answer).strip() == '':
+        return "Sorry, this information is not available yet"
+    answer_str = str(answer)
+    placeholders = re.findall(r'\{(\w+)\}', answer_str)
+    if not placeholders: return answer_str
+    if db_data is None:
+        return "To get this specific information, please ensure you are logged in or have provided your user ID."
+    missing_count = 0; replacements_made = 0
+    for placeholder in set(placeholders):
+        key = placeholder.strip()
+        value = db_data.get(key)
+        if value is None or (isinstance(value, float) and pd.isna(value)) or str(value).strip() == '':
+            answer_str = answer_str.replace(f'{{{key}}}', "not available")
+            missing_count += 1
+        else:
+            answer_str = answer_str.replace(f'{{{key}}}', str(value))
+            replacements_made +=1
+    if missing_count == len(placeholders) and len(placeholders) > 0 :
+        return "Sorry, some specific details for you are not available at the moment."
+    if "not available" in answer_str.lower() and replacements_made < len(placeholders):
+         if answer_str == "not available" and len(placeholders) == 1:
+             return "Sorry, this information is not available yet."
+    if re.search(r'\{(\w+)\}', answer_str):
+        logger.warning(f"Unresolved placeholders remain after replacement attempt: {answer_str}")
+        answer_str = re.sub(r'\{(\w+)\}', "a specific detail", answer_str)
+        if "a specific detail" in answer_str and not "Sorry" in answer_str:
+            return "Sorry, I couldn't retrieve all the specific details for this answer. " + answer_str
+        return "Sorry, I couldn't retrieve all the specific details for this answer. Some information has been generalized."
+    return answer_str
+
+# --- Non-Streaming Logic (Preserved from original) ---
+def get_hybrid_response_logic_non_streaming(user_query: str, session_id: str, user_id: Optional[str], chat_history: Optional[List[Dict]] = None) -> Dict[str, Any]:
+    global rag_system
+
+    if not user_query: return {'error': 'No query provided'}
+    if not session_id: return {'error': 'session_id is required'}
+
+    personal_db_data = personal_data_monitor.get_data(user_id) if user_id else None
+
+    # MODIFIED: Capture indices from the search
+    conf_greet, q_greet, a_greet, img_greet, idx_greet = embedding_manager.find_best_answers(user_query, 'greetings', top_n=1)
+    conf_pers, q_pers, a_pers, img_pers, idx_pers = embedding_manager.find_best_answers(user_query, 'personal', top_n=RELATED_QUESTIONS_TO_SHOW)
+    conf_gen, q_gen, a_gen, img_gen, idx_gen = embedding_manager.find_best_answers(user_query, 'general', top_n=RELATED_QUESTIONS_TO_SHOW)
+
+    all_csv_candidate_answers = []
+    if conf_greet and conf_greet[0] >= HIGH_CONFIDENCE_THRESHOLD:
+        all_csv_candidate_answers.append({'question': q_greet[0], 'answer': a_greet[0], 'image': img_greet[0] if img_greet else None, 'confidence': conf_greet[0], 'source_type': 'greetings', 'original_index': idx_greet[0]})
+    if conf_pers:
+        # MODIFIED: Add original_index to candidates
+        for c, q, a, img, idx in zip(conf_pers, q_pers, a_pers, img_pers, idx_pers):
+            processed_a = replace_placeholders_in_answer(a, personal_db_data)
+            if not ("Sorry, this information is not available yet" in processed_a or "To get this specific information" in processed_a):
+                all_csv_candidate_answers.append({'question': q, 'answer': processed_a, 'image': img, 'confidence': c, 'source_type': 'personal', 'original_index': idx})
+    if conf_gen:
+        # MODIFIED: Add original_index to candidates
+        for c, q, a, img, idx in zip(conf_gen, q_gen, a_gen, img_gen, idx_gen):
+            if not (pd.isna(a) or str(a).strip() == '' or str(a).lower() == 'nan'):
+                all_csv_candidate_answers.append({'question': q, 'answer': str(a), 'image': img, 'confidence': c, 'source_type': 'general', 'original_index': idx})
+
+    all_csv_candidate_answers.sort(key=lambda x: x['confidence'], reverse=True)
+
+    related_questions_list = []
+
+    if all_csv_candidate_answers:
+        best_csv_match = all_csv_candidate_answers[0]
+        is_direct_csv_answer = False
+        source_name = ""
+        
+        # MODIFIED: Use new configurable threshold for LLM formatting
+        best_source_type = best_csv_match['source_type']
+        best_confidence = best_csv_match['confidence']
+
+        if best_source_type == 'greetings' and best_confidence >= HIGH_CONFIDENCE_THRESHOLD:
+            is_direct_csv_answer = True
+            source_name = 'greetings_qa'
+        elif best_source_type in ['personal', 'general'] and best_confidence >= LLM_FORMATTER_CONFIDENCE_THRESHOLD:
+            is_direct_csv_answer = True
+            source_name = f"{best_source_type}_qa"
+
+        if is_direct_csv_answer:
+            # MODIFICATION START: Reroute high-confidence matches to the LLM for formatting
+            best_match_source = best_csv_match['source_type']
+            
+            # For greetings, we still provide a direct answer without LLM formatting
+            if best_match_source == 'greetings':
+                response_data = {'query': user_query, 'answer': best_csv_match['answer'], 'confidence': best_csv_match['confidence'], 'original_question': best_csv_match['question'], 'source': source_name}
+                if best_csv_match.get('image'):
+                     response_data['image_url'] = url_for('static', filename=best_csv_match['image'], _external=True)
+            else:
+                # For 'personal' and 'general', use the LLM to format the answer from the full row
+                best_match_index = best_csv_match['original_index']
+                
+                # Retrieve the full row from the original dataframe stored in the embedding manager
+                original_df = embedding_manager.embeddings[best_match_source].df_qa
+                matched_row_data = original_df.iloc[best_match_index]
+                
+                # Format the row data as a string context for the LLM
+                # We drop the 'Question' column as it's a duplicate of 'Pregunta' and not needed in the context
+                context_dict = matched_row_data.drop('Question', errors='ignore').to_dict()
+                context_str = "\n".join([f"'{key}': '{value}'" for key, value in context_dict.items() if pd.notna(value) and str(value).strip() != ''])
+
+                # Call the LLM to generate a conversational answer based on the row data
+                final_answer = get_answer_from_context(
+                    question=user_query,
+                    context=context_str,
+                    system_prompt=QA_FORMATTER_PROMPT
+                )
+                
+                response_data = {
+                    'query': user_query, 
+                    'answer': final_answer, 
+                    'confidence': best_csv_match['confidence'], 
+                    'original_question': best_csv_match['question'], 
+                    'source': f'{source_name}_llm_formatted'
+                }
+                if best_csv_match.get('image'):
+                     response_data['image_url'] = url_for('static', filename=best_csv_match['image'], _external=True)
+
+            # MODIFICATION END
+
+            for i, cand_q in enumerate(all_csv_candidate_answers):
+                if i == 0: continue
+                if cand_q['source_type'] != 'greetings':
+                     related_questions_list.append({'question': cand_q['question'], 'answer': cand_q['answer'], 'match': cand_q['confidence']})
+                if len(related_questions_list) >= RELATED_QUESTIONS_TO_SHOW: break
+            response_data['related_questions'] = related_questions_list
+            store_chat_history(session_id, user_id, user_query, response_data)
+            return response_data
+
+    if rag_system and rag_system.retriever:
+        logger.info(f"Attempting FAISS RAG query for: {user_query[:50]}...")
+        rag_result = rag_system.invoke(user_query) # Use invoke for non-streaming
+        rag_answer = rag_result.get("answer")
+        
+        if rag_answer and "the provided bibliography does not contain specific information" not in rag_answer.lower():
+            logger.info(f"FAISS RAG system provided a valid answer: {rag_answer[:100]}...")
+            response_data = {
+                'query': user_query, 'answer': rag_answer, 'confidence': 85,
+                'source': 'document_rag_faiss', 'related_questions': [],
+                'document_sources_details': rag_result.get("cited_source_details")
+            }
+            store_chat_history(session_id, user_id, user_query, response_data)
+            return response_data
+
+    logger.info(f"No high-confidence answer. Using Groq fallback.")
+    chat_history_messages_for_groq = chat_history if chat_history is not None else get_formatted_chat_history(session_id)
+    groq_context = {'current_query': user_query, 'chat_history': chat_history_messages_for_groq, 'qa_related_info': ""}
+    groq_stream = groq_bot_instance.stream_response(groq_context)
+    groq_answer = "".join([chunk for chunk in groq_stream])
+
+    response_data = {'query': user_query, 'answer': groq_answer, 'confidence': 75, 'source': 'groq_general_fallback', 'related_questions': []}
+    store_chat_history(session_id, user_id, user_query, response_data)
+    return response_data
+
+# --- Streaming Logic ---
+def generate_streaming_response(user_query: str, session_id: str, user_id: Optional[str], chat_history: Optional[List[Dict]] = None) -> Iterator[str]:
+    """
+    Handles the logic for generating a response and yields chunks of the response as a stream.
+    """
+    global rag_system
+    
+    personal_db_data = personal_data_monitor.get_data(user_id) if user_id else None
+    conf_greet, _, a_greet, _, idx_greet = embedding_manager.find_best_answers(user_query, 'greetings', top_n=1)
+    conf_pers, _, a_pers, _, idx_pers = embedding_manager.find_best_answers(user_query, 'personal', top_n=1)
+    conf_gen, _, a_gen, _, idx_gen = embedding_manager.find_best_answers(user_query, 'general', top_n=1)
+
+    # MODIFIED: Use new configurable threshold and logic for picking best candidate
+    candidates = []
+    # Greetings have their own threshold for a direct, non-LLM answer
+    if conf_greet and conf_greet[0] >= HIGH_CONFIDENCE_THRESHOLD:
+        candidates.append({'answer': a_greet[0], 'confidence': conf_greet[0], 'source': 'greetings', 'index': idx_greet[0]})
+    
+    # Personal and General QA have a stricter threshold to be sent to the LLM formatter
+    if conf_pers and conf_pers[0] >= LLM_FORMATTER_CONFIDENCE_THRESHOLD:
+        processed_a = replace_placeholders_in_answer(a_pers[0], personal_db_data)
+        # Only add candidate if placeholder replacement was successful
+        if not ("Sorry, this information is not available yet" in processed_a or "To get this specific information" in processed_a):
+            candidates.append({'answer': processed_a, 'confidence': conf_pers[0], 'source': 'personal', 'index': idx_pers[0]})
+
+    if conf_gen and conf_gen[0] >= LLM_FORMATTER_CONFIDENCE_THRESHOLD:
+        # Filter out empty/invalid answers
+        if not (pd.isna(a_gen[0]) or str(a_gen[0]).strip() == '' or str(a_gen[0]).lower() == 'nan'):
+            candidates.append({'answer': a_gen[0], 'confidence': conf_gen[0], 'source': 'general', 'index': idx_gen[0]})
+    
+    if candidates:
+        best_candidate = max(candidates, key=lambda x: x['confidence'])
+        best_source_type = best_candidate['source']
+        logger.info(f"High-confidence match from CSV source: {best_source_type}")
+
+        # If the best match is a greeting, yield it directly
+        if best_source_type == 'greetings':
+             yield best_candidate['answer']
+             return
+        
+        # Otherwise, the best match is 'personal' or 'general' and needs LLM formatting
+        original_df = embedding_manager.embeddings[best_source_type].df_qa
+        matched_row_data = original_df.iloc[best_candidate['index']]
+        context_dict = matched_row_data.drop('Question', errors='ignore').to_dict()
+        context_str = "\n".join([f"'{key}': '{value}'" for key, value in context_dict.items() if pd.notna(value) and str(value).strip() != ''])
+        
+        final_answer = get_answer_from_context(
+            question=user_query,
+            context=context_str,
+            system_prompt=QA_FORMATTER_PROMPT
+        )
+        yield final_answer
+        return
+
+    if rag_system and rag_system.retriever:
+        logger.info(f"Attempting to stream from FAISS RAG for: {user_query[:50]}...")
+        rag_stream = rag_system.stream(user_query)
+        first_chunk = next(rag_stream, None)
+        
+        if first_chunk and "the provided bibliography does not contain specific information" not in first_chunk.lower():
+            logger.info("FAISS RAG streaming valid answer...")
+            yield first_chunk
+            yield from rag_stream
+            return
+    
+    logger.info(f"No high-confidence CSV or RAG answer. Streaming from Groq fallback.")
+    chat_history_messages_for_groq = chat_history if chat_history is not None else get_formatted_chat_history(session_id)
+    groq_context = {'current_query': user_query, 'chat_history': chat_history_messages_for_groq, 'qa_related_info': ""}
+    yield from groq_bot_instance.stream_response(groq_context)
+
+def stream_formatter(logic_generator: Iterator[str], session_id: str, user_id: Optional[str], query: str) -> Iterator[str]:
+    """
+    Wraps raw text chunks into the Server-Sent Events (SSE) format and logs the full response at the end.
+    """
+    chunk_id = f"chatcmpl-{str(generate_uuid())}"
+    model_name = "MedicalAssisstantBot/v1"
+    full_response_chunks = []
+
+    for chunk in logic_generator:
+        if not chunk: continue
+        full_response_chunks.append(chunk)
+        response_json = {
+            "id": chunk_id, "object": "chat.completion.chunk", "created": int(time.time()), "model": model_name,
+            "choices": [{"index": 0, "delta": {"content": chunk}, "finish_reason": None}]
+        }
+        yield f"data: {json.dumps(response_json)}\n\n"
+
+    final_json = {
+        "id": chunk_id, "object": "chat.completion.chunk", "created": int(time.time()), "model": model_name,
+        "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+    }
+    yield f"data: {json.dumps(final_json)}\n\n"
+    yield "data: [DONE]\n\n"
+    
+    # After streaming is complete, log the full conversation to the database
+    full_response = "".join(full_response_chunks)
+    
+    # MODIFIED: Added print statement for full streamed response
+    print(f"\n--- STREAMED FULL RESPONSE ---")
+    print(full_response)
+    print(f"------------------------------\n")
+
+    history_manager.update_history(session_id, query, full_response)
+
+# --- Original Chat Endpoint (Preserved) ---
+@app.route('/chat-bot', methods=['POST'])
+def get_answer_hybrid():
+    data = request.json
+    user_query = data.get('query', '')
+    user_query = clean_html_from_text(user_query) # ADDED
+    user_id = data.get('user_id')
+    session_id = data.get('session_id')
+    
+    if not user_query or not session_id:
+        return jsonify({'error': 'query and session_id are required'}), 400
+
+    response_data = get_hybrid_response_logic_non_streaming(user_query, session_id, user_id, None)
+    return jsonify(response_data)
+
+# --- OpenAI Compatible Endpoints (Added) ---
+@app.route('/v1/models', methods=['GET'])
+def list_models():
+    model_data = {
+        "object": "list",
+        "data": [{"id": "MedicalAssisstantBot/v1", "object": "model", "created": int(time.time()), "owned_by": "user"}]
+    }
+    return jsonify(model_data)
+
+@app.route('/v1/chat/completions', methods=['POST'])
+def openai_compatible_chat_endpoint():
+    data = request.json
+    is_streaming = data.get("stream", False)
+    
+    messages = data.get("messages", [])
+    if not messages: return jsonify({"error": "No messages provided"}), 400
+        
+    user_query = messages[-1].get("content", "")
+    user_query = clean_html_from_text(user_query) # ADDED
+    chat_history = messages[:-1]
+    session_id = data.get("conversation_id", f"webui-session-{str(generate_uuid())}")
+    user_id = None
+
+    if is_streaming:
+        logic_generator = generate_streaming_response(user_query, session_id, user_id, chat_history)
+        return Response(stream_formatter(logic_generator, session_id, user_id, user_query), mimetype='text/event-stream')
+    else:
+        full_response_dict = get_hybrid_response_logic_non_streaming(user_query, session_id, user_id, chat_history)
+        response_content = full_response_dict.get("answer", "Sorry, an error occurred.")
+        
+        openai_response = {
+            "id": f"chatcmpl-{str(generate_uuid())}", "object": "chat.completion", "created": int(time.time()),
+            "model": "MedicalAssisstantBot/v1",
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": response_content}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+        }
+        history_manager.update_history(session_id, user_query, response_content)
+        return jsonify(openai_response)
+
+
+# --- Admin and Utility Routes (Unchanged) ---
+@app.route('/')
+def index_route():
+    template_to_render = 'chat-bot.html'
+    if not os.path.exists(os.path.join(app.root_path, 'templates', template_to_render)):
+        logger.warning(f"Template '{template_to_render}' not found. Serving basic message.")
+        return "Chatbot interface not found. Please ensure 'templates/chat-bot.html' exists.", 404
+    return render_template(template_to_render)
+
+@app.route('/admin/faiss_rag_status', methods=['GET'])
+@require_admin_auth
+def get_faiss_rag_status():
+    global rag_system
+    if not rag_system:
+        return jsonify({"error": "FAISS RAG system not initialized."}), 500
+    try:
+        status = {
+            "status": "Initialized" if rag_system.retriever else "Initialized (Retriever not ready)",
+            "index_storage_dir": rag_system.index_storage_dir,
+            "embedding_model": rag_system.embedding_model_name,
+            "groq_model": rag_system.groq_model_name,
+            "retriever_k": rag_system.retriever.k if rag_system.retriever else "N/A",
+            "processed_source_files": rag_system.processed_source_files,
+            "index_type": "FAISS",
+            "index_loaded_or_built": rag_system.vector_store is not None
+        }
+        if rag_system.vector_store and hasattr(rag_system.vector_store, 'index') and rag_system.vector_store.index:
+            try:
+                status["num_vectors_in_index"] = rag_system.vector_store.index.ntotal
+            except Exception:
+                status["num_vectors_in_index"] = "N/A (Could not get count)"
+        else:
+            status["num_vectors_in_index"] = "N/A (Vector store or index not available)"
+        return jsonify(status)
+    except Exception as e:
+        logger.error(f"Error getting FAISS RAG status: {e}", exc_info=True)
+        return jsonify({"error": str(e)}), 500
+
+# NEW FUNCTION: Endpoint to download the combined QA databases as an Excel file
+@app.route('/admin/download_qa_database', methods=['GET'])
+@require_admin_auth
+def download_qa_database():
+    try:
+        output = io.BytesIO()
+        with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+            # Safely access the dataframes from the embedding manager
+            if embedding_manager.embeddings['general'] and embedding_manager.embeddings['general'].df_qa is not None:
+                embedding_manager.embeddings['general'].df_qa.to_excel(writer, sheet_name='General_QA', index=False)
+            
+            if embedding_manager.embeddings['personal'] and embedding_manager.embeddings['personal'].df_qa is not None:
+                embedding_manager.embeddings['personal'].df_qa.to_excel(writer, sheet_name='Personal_QA', index=False)
+
+            if embedding_manager.embeddings['greetings'] and embedding_manager.embeddings['greetings'].df_qa is not None:
+                embedding_manager.embeddings['g greetings'].df_qa.to_excel(writer, sheet_name='Greetings', index=False)
+        
+        output.seek(0)
+        
+        return send_file(
+            output,
+            mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            as_attachment=True,
+            download_name=f'qa_database_{datetime.now().strftime("%Y%m%d")}.xlsx'
+        )
+    except Exception as e:
+        logger.error(f"Error generating QA database file: {e}", exc_info=True)
+        return jsonify({'error': 'Failed to generate QA database file'}), 500
+
+@app.route('/admin/rebuild_faiss_index', methods=['POST'])
+@require_admin_auth
+def rebuild_faiss_index_route():
+    global rag_system
+    logger.info("Admin request to rebuild FAISS RAG index received. Starting two-step process.")
+
+    logger.info("Step 1: Running chunker.py to pre-process source documents.")
+    chunker_script_path = os.path.join(_APP_BASE_DIR, 'chunker.py')
+    chunked_json_output_path = os.path.join(RAG_STORAGE_PARENT_DIR, RAG_CHUNKED_SOURCES_FILENAME)
+
+    os.makedirs(TEXT_EXTRACTIONS_DIR, exist_ok=True)
+
+    if not os.path.exists(chunker_script_path):
+        logger.error(f"Chunker script not found at '{chunker_script_path}'. Aborting rebuild.")
+        return jsonify({"error": f"chunker.py not found. Cannot proceed with rebuild."}), 500
+
+    command = [
+        sys.executable,
+        chunker_script_path,
+        '--sources-dir', RAG_SOURCES_DIR,
+        '--output-file', chunked_json_output_path,
+        '--text-output-dir', TEXT_EXTRACTIONS_DIR
+    ]
+
+    try:
+        process = subprocess.run(command, capture_output=True, text=True, check=True)
+        logger.info("Chunker script executed successfully.")
+        logger.info(f"Chunker stdout:\n{process.stdout}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Chunker script failed with exit code {e.returncode}.")
+        logger.error(f"Chunker stderr:\n{e.stderr}")
+        return jsonify({"error": "Step 1 (Chunking) failed.", "details": e.stderr}), 500
+    except Exception as e:
+        logger.error(f"An unexpected error occurred while running the chunker script: {e}", exc_info=True)
+        return jsonify({"error": f"An unexpected error occurred during the chunking step: {str(e)}"}), 500
+
+    logger.info("Step 2: Rebuilding FAISS index from the newly generated chunks.")
+    try:
+        new_rag_system_instance = initialize_and_get_rag_system(force_rebuild=True)
+
+        if new_rag_system_instance and new_rag_system_instance.vector_store:
+            rag_system = new_rag_system_instance
+            logger.info("FAISS RAG index rebuild completed and new RAG system instance is active.")
+            updated_status_response = get_faiss_rag_status()
+            return jsonify({"message": "FAISS RAG index rebuild completed.", "status": updated_status_response.get_json()}), 200
+        else:
+            logger.error("FAISS RAG index rebuild failed during the indexing phase.")
+            return jsonify({"error": "Step 2 (Indexing) failed. Check logs."}), 500
+
+    except Exception as e:
+        logger.error(f"Error during admin FAISS index rebuild (indexing phase): {e}", exc_info=True)
+        return jsonify({"error": f"Failed to rebuild index during indexing phase: {str(e)}"}), 500
+
+@app.route('/db/status', methods=['GET'])
+@require_admin_auth
+def get_personal_db_status():
+    try:
+        status_info = {
+            'personal_data_csv_monitor_status': 'running',
+            'file_exists': os.path.exists(personal_data_monitor.database_path),
+            'data_loaded': personal_data_monitor.df is not None, 'last_update': None
+        }
+        if status_info['file_exists'] and os.path.getmtime(personal_data_monitor.database_path) is not None:
+            status_info['last_update'] = datetime.fromtimestamp(os.path.getmtime(personal_data_monitor.database_path)).isoformat()
+        return jsonify(status_info)
+    except Exception as e: return jsonify({'status': 'error', 'error': str(e)}), 500
+
+@app.route('/report', methods=['GET'])
+@require_admin_auth
+def download_report():
+    try:
+        if not os.path.exists(CHAT_LOG_FILE) or os.path.getsize(CHAT_LOG_FILE) == 0:
+            return jsonify({'error': 'No chat history available.'}), 404
+        return send_file(CHAT_LOG_FILE, mimetype='text/csv', as_attachment=True, download_name=f'chat_history_{datetime.now().strftime("%Y%m%d_%H%M%S")}.csv')
+    except Exception as e:
+        logger.error(f"Error downloading report: {e}", exc_info=True)
+        return jsonify({'error': 'Failed to generate report'}), 500
+
+@app.route('/create-session', methods=['POST'])
+def create_session_route():
+    try:
+        session_id = str(generate_uuid())
+        logger.info(f"New session created: {session_id}")
+        return jsonify({'status': 'success', 'session_id': session_id}), 200
+    except Exception as e:
+        logger.error(f"Session creation error: {e}", exc_info=True)
+        return jsonify({'status': 'error', 'message': str(e)}), 500
+
+@app.route('/version', methods=['GET'])
+def get_version_route():
+    return jsonify({'version': '3.9.8-Env-Chat-History'}), 200 # Updated version
+
+@app.route('/clear-history', methods=['POST'])
+def clear_session_history_route():
+    data = request.json
+    session_id = data.get('session_id')
+    if not session_id: 
+        return jsonify({'status': 'error', 'message': 'session_id is required'}), 400
+    history_manager.clear_history(session_id)
+    return jsonify({'status': 'success', 'message': f'History cleared for session {session_id}'})
+
+# --- App Cleanup and Startup ---
+def cleanup_application():
+    if personal_data_monitor: personal_data_monitor.stop()
+    logger.info("Application cleanup finished.")
+atexit.register(cleanup_application)
+
+def load_qa_data_on_startup():
+    global embedding_manager
+    # MODIFIED: Added print statement
+    print("\n--- Loading QA Source Files ---")
+    try:
+        general_qa_path = os.path.join(RAG_SOURCES_DIR, 'general_qa.csv')
+        personal_qa_path = os.path.join(RAG_SOURCES_DIR, 'personal_qa.csv')
+        greetings_qa_path = os.path.join(RAG_SOURCES_DIR, 'greetings.csv')
+
+        general_qa_df = pd.DataFrame(columns=['Question', 'Answer', 'Image'])
+        personal_qa_df = pd.DataFrame(columns=['Question', 'Answer', 'Image'])
+        greetings_qa_df = pd.DataFrame(columns=['Question', 'Answer', 'Image'])
+
+        if os.path.exists(general_qa_path):
+            try: 
+                general_qa_df = pd.read_csv(general_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(general_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading general_qa.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'general_qa.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        if os.path.exists(personal_qa_path):
+            try: 
+                personal_qa_df = pd.read_csv(personal_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(personal_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading personal_qa.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'personal_qa.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        if os.path.exists(greetings_qa_path):
+            try: 
+                greetings_qa_df = pd.read_csv(greetings_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(greetings_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading greetings.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'greetings.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        logger.info(f"Scanning for additional QA sources (.xlsx) in '{RAG_SOURCES_DIR}'...")
+        if os.path.isdir(RAG_SOURCES_DIR):
+            xlsx_files_found = [f for f in os.listdir(RAG_SOURCES_DIR) if f.endswith('.xlsx') and os.path.isfile(os.path.join(RAG_SOURCES_DIR, f))]
+            
+            if xlsx_files_found:
+                all_general_dfs = [general_qa_df] if not general_qa_df.empty else []
+                for xlsx_file in xlsx_files_found:
+                    try:
+                        xlsx_path = os.path.join(RAG_SOURCES_DIR, xlsx_file)
+                        logger.info(f"Processing XLSX source file: {xlsx_file}")
+                        df_excel = pd.read_excel(xlsx_path)
+
+                        # MODIFIED: New logic to preserve all columns and handle dynamic headers
+                        if 'Pregunta' in df_excel.columns and 'Respuesta' in df_excel.columns:
+                            logger.info(f"Found 'Pregunta' and 'Respuesta' in {xlsx_file}. Preserving all columns.")
+                            # The 'Question' column is required by the EmbeddingManager for semantic search.
+                            # We create it from 'Pregunta' but keep all original columns.
+                            df_excel['Question'] = df_excel['Pregunta']
+                            all_general_dfs.append(df_excel)
+                            print(f"- Loaded and processing: {xlsx_file}")
+                        else:
+                            logger.warning(f"Skipping XLSX file '{xlsx_file}' as it lacks the required 'Pregunta' and 'Respuesta' columns.")
+                    except Exception as e_xlsx:
+                        logger.error(f"Error processing XLSX file '{xlsx_file}': {e_xlsx}")
+                
+                if len(all_general_dfs) > 0:
+                    general_qa_df = pd.concat(all_general_dfs, ignore_index=True)
+                    logger.info(f"Successfully merged data from {len(xlsx_files_found)} XLSX file(s) into the general QA set.")
+        else:
+            logger.warning(f"Sources directory '{RAG_SOURCES_DIR}' not found. Cannot scan for additional QA files.")
+
+        dataframes_to_process = {
+            "general": general_qa_df,
+            "personal": personal_qa_df,
+            "greetings": greetings_qa_df
+        }
+
+        for df_name, df_val in dataframes_to_process.items():
+            if df_val.empty: continue
+
+            # Normalize text in all columns to prevent issues
+            for col in df_val.columns:
+                 if not df_val[col].isnull().all():
+                     df_val[col] = df_val[col].astype(str).apply(normalize_text)
+
+            # Ensure 'Question' column exists for embedding manager compatibility
+            if 'Question' not in df_val.columns:
+                # For CSVs that might not have 'Pregunta' but have 'Question'
+                if 'Question' in df_val.columns:
+                     pass # Already exists
+                else:
+                    df_val['Question'] = None
+                    logger.warning(f"'Question' column missing in {df_name} data. Added empty column.")
+
+        embedding_manager.update_embeddings(
+            dataframes_to_process["general"],
+            dataframes_to_process["personal"],
+            dataframes_to_process["greetings"]
+        )
+        logger.info("CSV & XLSX QA data loaded and embeddings initialized.")
+
+    except Exception as e:
+        logger.critical(f"CRITICAL: Error loading or processing QA data: {e}. Semantic QA may not function.", exc_info=True)
+    # MODIFIED: Added print statement
+    print("-----------------------------\n")
+
+if __name__ == '__main__':
+    for folder_path in [os.path.join(_APP_BASE_DIR, 'templates'),
+                        os.path.join(_APP_BASE_DIR, 'static'),
+                        TEXT_EXTRACTIONS_DIR]:
+        os.makedirs(folder_path, exist_ok=True)
+
+    load_qa_data_on_startup()
+    initialize_chat_log()
+
+    logger.info("Attempting to initialize RAG system from llm_handling module...")
+    rag_system = initialize_and_get_rag_system()
+    if rag_system:
+        logger.info("RAG system initialized successfully via llm_handling module.")
+    else:
+        logger.warning("RAG system failed to initialize. Document RAG functionality will be unavailable.")
+
+    logger.info(f"Flask application starting with Hybrid RAG on {FLASK_APP_HOST}:{FLASK_APP_PORT} Debug: {FLASK_DEBUG_MODE}...")
+    if not FLASK_DEBUG_MODE:
+        werkzeug_log = logging.getLogger('werkzeug')
+        werkzeug_log.setLevel(logging.ERROR)
+
+    app.run(host=FLASK_APP_HOST, port=FLASK_APP_PORT, debug=FLASK_DEBUG_MODE, use_reloader=False)

chunker.py

@@ -0,0 +1,189 @@
+import os
+import logging
+import json
+import argparse
+from typing import List, Dict, Optional
+
+from pypdf import PdfReader
+import docx as python_docx
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+# --- Logging Setup ---
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# --- Text Extraction Helper Functions ---
+# Note: These are duplicated from llm_handling.py to make this a standalone script.
+def extract_text_from_file(file_path: str, file_type: str) -> Optional[str]:
+    logger.info(f"Extracting text from {file_type.upper()} file: {os.path.basename(file_path)}")
+    text_content = None
+    try:
+        if file_type == 'pdf':
+            reader = PdfReader(file_path)
+            text_content = "".join(page.extract_text() + "\n" for page in reader.pages if page.extract_text())
+        elif file_type == 'docx':
+            doc = python_docx.Document(file_path)
+            text_content = "\n".join(para.text for para in doc.paragraphs if para.text)
+        elif file_type == 'txt':
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text_content = f.read()
+        else:
+            logger.warning(f"Unsupported file type for text extraction: {file_type} for file {os.path.basename(file_path)}")
+            return None
+
+        if not text_content or not text_content.strip():
+            logger.warning(f"No text content extracted from {os.path.basename(file_path)}")
+            return None
+        return text_content.strip()
+    except Exception as e:
+        logger.error(f"Error extracting text from {os.path.basename(file_path)} ({file_type.upper()}): {e}", exc_info=True)
+        return None
+
+SUPPORTED_EXTENSIONS = {
+    'pdf': lambda path: extract_text_from_file(path, 'pdf'),
+    'docx': lambda path: extract_text_from_file(path, 'docx'),
+    'txt': lambda path: extract_text_from_file(path, 'txt'),
+}
+
+def process_sources_and_create_chunks(
+    sources_dir: str,
+    output_file: str,
+    chunk_size: int = 1000,
+    chunk_overlap: int = 150,
+    text_output_dir: Optional[str] = None  # MODIFIED: Added optional parameter
+) -> None:
+    """
+    Scans a directory for source files, extracts text, splits it into chunks,
+    and saves the chunks to a single JSON file.
+    Optionally saves the raw extracted text to a specified directory.
+    """
+    if not os.path.isdir(sources_dir):
+        logger.error(f"Source directory not found: '{sources_dir}'")
+        raise FileNotFoundError(f"Source directory not found: '{sources_dir}'")
+
+    logger.info(f"Starting chunking process. Sources: '{sources_dir}', Output: '{output_file}'")
+    
+    # MODIFIED: Create text output directory if provided
+    if text_output_dir:
+        os.makedirs(text_output_dir, exist_ok=True)
+        logger.info(f"Will save raw extracted text to: '{text_output_dir}'")
+
+    all_chunks_for_json: List[Dict] = []
+    processed_files_count = 0
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+    for filename in os.listdir(sources_dir):
+        file_path = os.path.join(sources_dir, filename)
+        if not os.path.isfile(file_path):
+            continue
+
+        file_ext = filename.split('.')[-1].lower()
+        if file_ext not in SUPPORTED_EXTENSIONS:
+            logger.debug(f"Skipping unsupported file: {filename}")
+            continue
+
+        logger.info(f"Processing source file: {filename}")
+        text_content = SUPPORTED_EXTENSIONS[file_ext](file_path)
+
+        if text_content:
+            # MODIFIED: Save the raw text to a file if directory is specified
+            if text_output_dir:
+                try:
+                    text_output_path = os.path.join(text_output_dir, f"{filename}.txt")
+                    with open(text_output_path, 'w', encoding='utf-8') as f_text:
+                        f_text.write(text_content)
+                    logger.info(f"Saved extracted text for '{filename}' to '{text_output_path}'")
+                except Exception as e_text_save:
+                    logger.error(f"Could not save extracted text for '{filename}': {e_text_save}")
+
+            chunks = text_splitter.split_text(text_content)
+            if not chunks:
+                logger.warning(f"No chunks generated from {filename}. Skipping.")
+                continue
+
+            for i, chunk_text in enumerate(chunks):
+                chunk_data = {
+                    "page_content": chunk_text,
+                    "metadata": {
+                        "source_document_name": filename,
+                        "chunk_index": i,
+                        "full_location": f"{filename}, Chunk {i+1}"
+                    }
+                }
+                all_chunks_for_json.append(chunk_data)
+
+            processed_files_count += 1
+        else:
+            logger.warning(f"Could not extract text from {filename}. Skipping.")
+
+    if not all_chunks_for_json:
+        logger.warning(f"No processable documents found or no text extracted in '{sources_dir}'. JSON file will be empty.")
+
+    output_dir = os.path.dirname(output_file)
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(all_chunks_for_json, f, indent=2)
+
+    logger.info(f"Chunking complete. Processed {processed_files_count} files.")
+    logger.info(f"Created a total of {len(all_chunks_for_json)} chunks.")
+    logger.info(f"Chunked JSON output saved to: {output_file}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Process source documents into a JSON file of text chunks for RAG.")
+    parser.add_argument(
+        '--sources-dir',
+        type=str,
+        required=True,
+        help="The directory containing source files (PDFs, DOCX, TXT)."
+    )
+    parser.add_argument(
+        '--output-file',
+        type=str,
+        required=True,
+        help="The full path for the output JSON file containing the chunks."
+    )
+    # MODIFIED: Added new optional argument
+    parser.add_argument(
+        '--text-output-dir',
+        type=str,
+        default=None,
+        help="Optional: The directory to save raw extracted text files for debugging."
+    )
+    parser.add_argument(
+        '--chunk-size',
+        type=int,
+        default=1000,
+        help="The character size for each text chunk."
+    )
+    parser.add_argument(
+        '--chunk-overlap',
+        type=int,
+        default=150,
+        help="The character overlap between consecutive chunks."
+    )
+
+    args = parser.parse_args()
+
+    try:
+        process_sources_and_create_chunks(
+            sources_dir=args.sources_dir,
+            output_file=args.output_file,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+            text_output_dir=args.text_output_dir  # MODIFIED: Pass argument
+        )
+    except Exception as e:
+        logger.critical(f"A critical error occurred during the chunking process: {e}", exc_info=True)
+        exit(1)
+
+if __name__ == "__main__":
+    main()

env

@@ -0,0 +1,31 @@
+
+FLASK_HOST=0.0.0.0
+FLASK_PORT=7860
+# FLASK_DEBUG="False"  # Gradio uses debug in launch()
+
+# --- Groq LLM Models ---
+GROQ_FALLBACK_MODEL=qwen/qwen3-32b
+GROQ_AUXILIARY_MODEL=llama3-8b-8192
+
+RAG_LLM_MODEL=qwen/qwen3-32b
+RAG_TEMPERATURE=0.1
+
+# --- RAG System Configuration ---
+# Embedding model for RAG
+RAG_EMBEDDING_MODEL=all-MiniLM-L6-v2
+
+# Whether to use GPU for RAG embeddings (if available and faiss-gpu is installed)
+
+RAG_EMBEDDING_GPU=false
+
+# Whether to attempt loading an existing RAG index on startup.
+RAG_LOAD_INDEX=true
+
+# Default number of documents the RAG retriever should fetch.
+RAG_RETRIEVER_K=5
+
+GDRIVE_SOURCES_ENABLED=False
+GDRIVE_FOLDER_URL=&HBGGtZ4TJA
+
+LLM_FORMATTER_CONFIDENCE_THRESHOLD=95
+CHAT_HISTORY_TO_SEND=5

faiss_storage/faiss_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c627f912c7eead10f1ed14a68211eaad41b98e8920deb999f79f8671dc01979
+size 3640365

faiss_storage/faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7f4e06d1c5d0de27cae7d9a9328c94073faa58705ceea28242166dca11c581e
+size 2569631

gitattributes

@@ -0,0 +1,59 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_storage/faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]appendix[[:space:]]4.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]1.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]10.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]11.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]12.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]13.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]14.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]15.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]16.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]17.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]18.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]19.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]2.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]20.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]21.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]22.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]3.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]4.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]5.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]6.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]7.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]8.pdf filter=lfs diff=lfs merge=lfs -text
+sources/Endodontics[[:space:]]cap[[:space:]]9.pdf filter=lfs diff=lfs merge=lfs -text

llm_handling.py

@@ -0,0 +1,542 @@
+import os
+import logging
+import json
+from typing import List, Dict, Tuple, Optional, Any, Iterator
+import shutil
+import re
+import time
+import requests
+import zipfile
+import tempfile
+import gdown
+
+import torch
+from sentence_transformers import SentenceTransformer
+from pypdf import PdfReader
+import docx as python_docx
+
+from llama_index.core.llms import ChatMessage
+from llama_index.llms.groq import Groq as LlamaIndexGroqClient
+
+from langchain_groq import ChatGroq
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema import Document, BaseRetriever
+from langchain.callbacks.manager import CallbackManagerForRetrieverRun
+from langchain.schema.runnable import RunnablePassthrough, RunnableParallel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+# MODIFIED: Import the new prompt
+from system_prompts import RAG_SYSTEM_PROMPT, FALLBACK_SYSTEM_PROMPT, QA_FORMATTER_PROMPT
+
+logger = logging.getLogger(__name__)
+if not logger.handlers:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
+# --- Configuration Constants ---
+GROQ_API_KEY = os.getenv('BOT_API_KEY')
+if not GROQ_API_KEY:
+    logger.critical("CRITICAL: BOT_API_KEY environment variable not found. Services will fail.")
+
+FALLBACK_LLM_MODEL_NAME = os.getenv("GROQ_FALLBACK_MODEL", "llama-3.1-70b-versatile")
+# ADDED: New constant for the auxiliary model
+AUXILIARY_LLM_MODEL_NAME = os.getenv("GROQ_AUXILIARY_MODEL", "llama3-8b-8192")
+_MODULE_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+RAG_FAISS_INDEX_SUBDIR_NAME = "faiss_index"
+RAG_STORAGE_PARENT_DIR = os.getenv("RAG_STORAGE_DIR", os.path.join(_MODULE_BASE_DIR, "faiss_storage"))
+RAG_SOURCES_DIR = os.getenv("SOURCES_DIR", os.path.join(_MODULE_BASE_DIR, "sources"))
+RAG_CHUNKED_SOURCES_FILENAME = "pre_chunked_sources.json"
+os.makedirs(RAG_SOURCES_DIR, exist_ok=True)
+os.makedirs(RAG_STORAGE_PARENT_DIR, exist_ok=True)
+RAG_EMBEDDING_MODEL_NAME = os.getenv("RAG_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+RAG_EMBEDDING_USE_GPU = os.getenv("RAG_EMBEDDING_GPU", "False").lower() == "true"
+RAG_LLM_MODEL_NAME = os.getenv("RAG_LLM_MODEL", "llama-3.1-70b-versatile")
+RAG_LLM_TEMPERATURE = float(os.getenv("RAG_TEMPERATURE", 0.0))
+RAG_LOAD_INDEX_ON_STARTUP = os.getenv("RAG_LOAD_INDEX", "True").lower() == "true"
+RAG_DEFAULT_RETRIEVER_K = int(os.getenv("RAG_RETRIEVER_K", 3))
+GDRIVE_SOURCES_ENABLED = os.getenv("GDRIVE_SOURCES_ENABLED", "False").lower() == "true"
+GDRIVE_FOLDER_ID_OR_URL = os.getenv("GDRIVE_FOLDER_URL")
+
+# --- Text Extraction Helper Function ---
+def extract_text_from_file(file_path: str, file_type: str) -> Optional[str]:
+    logger.info(f"Extracting text from {file_type.upper()} file: {os.path.basename(file_path)}")
+    try:
+        if file_type == 'pdf':
+            reader = PdfReader(file_path)
+            return "".join(page.extract_text() + "\n" for page in reader.pages if page.extract_text())
+        elif file_type == 'docx':
+            doc = python_docx.Document(file_path)
+            return "\n".join(para.text for para in doc.paragraphs if para.text)
+        elif file_type == 'txt':
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                return f.read()
+        logger.warning(f"Unsupported file type for text extraction: {file_type}")
+        return None
+    except Exception as e:
+        logger.error(f"Error extracting text from {os.path.basename(file_path)}: {e}", exc_info=True)
+        return None
+
+FAISS_RAG_SUPPORTED_EXTENSIONS = {'pdf': 'pdf', 'docx': 'docx', 'txt': 'txt'}
+
+# --- FAISS RAG System ---
+class FAISSRetrieverWithScore(BaseRetriever):
+    vectorstore: FAISS
+    k: int = RAG_DEFAULT_RETRIEVER_K
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        docs_and_scores = self.vectorstore.similarity_search_with_score(query, k=self.k)
+        relevant_docs = []
+        for doc, score in docs_and_scores:
+            doc.metadata["retrieval_score"] = float(score)
+            relevant_docs.append(doc)
+        return relevant_docs
+
+class KnowledgeRAG:
+    def __init__(
+        self,
+        index_storage_dir: str,
+        embedding_model_name: str,
+        groq_model_name_for_rag: str,
+        use_gpu_for_embeddings: bool,
+        groq_api_key_for_rag: str,
+        temperature: float,
+    ):
+        self.logger = logging.getLogger(__name__ + ".KnowledgeRAG")
+        self.index_storage_dir = index_storage_dir
+        self.embedding_model_name = embedding_model_name
+        self.groq_model_name = groq_model_name_for_rag
+        self.temperature = temperature
+
+        device = "cuda" if use_gpu_for_embeddings and torch.cuda.is_available() else "cpu"
+        self.logger.info(f"Initializing Hugging Face embedding model: {self.embedding_model_name} on device: {device}")
+        try:
+            self.embeddings = HuggingFaceEmbeddings(
+                model_name=self.embedding_model_name,
+                model_kwargs={"device": device},
+                encode_kwargs={"normalize_embeddings": True}
+            )
+        except Exception as e:
+            self.logger.critical(f"Failed to load embedding model: {e}", exc_info=True)
+            raise
+
+        self.logger.info(f"Initializing Langchain ChatGroq LLM for RAG: {self.groq_model_name}")
+        if not groq_api_key_for_rag:
+            raise ValueError("Groq API Key for RAG is missing.")
+        try:
+            self.llm = ChatGroq(
+                temperature=self.temperature,
+                groq_api_key=groq_api_key_for_rag,
+                model_name=self.groq_model_name
+            )
+        except Exception as e:
+            self.logger.critical(f"Failed to initialize Langchain ChatGroq LLM: {e}", exc_info=True)
+            raise
+
+        self.vector_store: Optional[FAISS] = None
+        self.retriever: Optional[FAISSRetrieverWithScore] = None
+        self.rag_chain = None
+        self.processed_source_files: List[str] = []
+
+    def build_index_from_source_files(self, source_folder_path: str, k: int = RAG_DEFAULT_RETRIEVER_K):
+        all_docs_for_vectorstore: List[Document] = []
+        processed_files_this_build: List[str] = []
+        pre_chunked_json_path = os.path.join(self.index_storage_dir, RAG_CHUNKED_SOURCES_FILENAME)
+
+        if os.path.exists(pre_chunked_json_path):
+            self.logger.info(f"Loading documents from pre-chunked file: {pre_chunked_json_path}")
+            try:
+                with open(pre_chunked_json_path, 'r', encoding='utf-8') as f:
+                    chunk_data_list = json.load(f)
+                source_filenames = set()
+                for chunk_data in chunk_data_list:
+                    doc = Document(page_content=chunk_data.get("page_content", ""), metadata=chunk_data.get("metadata", {}))
+                    all_docs_for_vectorstore.append(doc)
+                    if 'source_document_name' in doc.metadata:
+                        source_filenames.add(doc.metadata['source_document_name'])
+                processed_files_this_build = sorted(list(source_filenames))
+            except Exception as e:
+                self.logger.error(f"Error processing pre-chunked JSON, falling back to raw files: {e}")
+                all_docs_for_vectorstore.clear()
+
+        if not all_docs_for_vectorstore:
+            self.logger.info(f"Processing raw files from '{source_folder_path}' to build index.")
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+            for filename in os.listdir(source_folder_path):
+                file_path = os.path.join(source_folder_path, filename)
+                file_ext = filename.split('.')[-1].lower()
+                if os.path.isfile(file_path) and file_ext in FAISS_RAG_SUPPORTED_EXTENSIONS:
+                    text_content = extract_text_from_file(file_path, file_ext)
+                    if text_content:
+                        chunks = text_splitter.split_text(text_content)
+                        for i, chunk_text in enumerate(chunks):
+                            metadata = {"source_document_name": filename, "chunk_index": i}
+                            all_docs_for_vectorstore.append(Document(page_content=chunk_text, metadata=metadata))
+                        processed_files_this_build.append(filename)
+
+        if not all_docs_for_vectorstore:
+            self.logger.warning(f"No processable PDF/DOCX/TXT documents found in '{source_folder_path}'. RAG index will only contain other sources if available.")
+
+
+        self.processed_source_files = processed_files_this_build
+        
+        # This print statement is kept for console visibility on startup/rebuild
+        print("\n--- Document Files Used for RAG Index ---")
+        if self.processed_source_files:
+            for filename in self.processed_source_files:
+                print(f"- {filename}")
+        else:
+            print("No PDF/DOCX/TXT source files were processed for the RAG index.")
+        print("---------------------------------------\n")
+
+        if not all_docs_for_vectorstore:
+            self.logger.warning("No documents to build FAISS index from. Skipping FAISS build.")
+            return
+
+        self.logger.info(f"Creating FAISS index from {len(all_docs_for_vectorstore)} document chunks...")
+        self.vector_store = FAISS.from_documents(all_docs_for_vectorstore, self.embeddings)
+        faiss_index_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        self.vector_store.save_local(faiss_index_path)
+        self.logger.info(f"FAISS index built and saved to '{faiss_index_path}'.")
+        self.retriever = FAISSRetrieverWithScore(vectorstore=self.vector_store, k=k)
+        self.setup_rag_chain()
+
+    def load_index_from_disk(self, k: int = RAG_DEFAULT_RETRIEVER_K):
+        faiss_index_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        if not os.path.isdir(faiss_index_path):
+            raise FileNotFoundError(f"FAISS index directory not found at '{faiss_index_path}'.")
+
+        self.logger.info(f"Loading FAISS index from: {faiss_index_path}")
+        self.vector_store = FAISS.load_local(
+            folder_path=faiss_index_path,
+            embeddings=self.embeddings,
+            allow_dangerous_deserialization=True
+        )
+        self.retriever = FAISSRetrieverWithScore(vectorstore=self.vector_store, k=k)
+        self.setup_rag_chain()
+
+    def format_docs(self, docs: List[Document]) -> str:
+        return "\n\n---\n\n".join([f"[Excerpt from {doc.metadata.get('source_document_name', 'N/A')}, Chunk {doc.metadata.get('chunk_index', 'N/A')}]\nContent:\n{doc.page_content}" for doc in docs])
+
+    def setup_rag_chain(self):
+        if not self.retriever or not self.llm:
+            raise RuntimeError("Retriever and LLM must be initialized.")
+        
+        prompt = ChatPromptTemplate.from_template(RAG_SYSTEM_PROMPT)
+
+        self.rag_chain = (
+            RunnableParallel(
+                context=(self.retriever | self.format_docs),
+                question=RunnablePassthrough()
+            )
+            | prompt
+            | self.llm
+            | StrOutputParser()
+        )
+        self.logger.info("RAG LCEL chain set up successfully with dental assistant persona.")
+
+    def invoke(self, query: str, top_k: Optional[int] = None) -> Dict[str, Any]:
+        if not self.rag_chain:
+            # MODIFIED: Changed severity
+            self.logger.warning("RAG system not fully initialized. Cannot invoke.")
+            return {"answer": "The provided bibliography does not contain specific information on this topic.", "source": "system_error", "cited_source_details": []}
+
+        if not query or not query.strip():
+            return {"answer": "Please provide a valid question.", "source": "system_error", "cited_source_details": []}
+
+        k_to_use = top_k if top_k is not None and top_k > 0 else self.retriever.k
+        self.logger.info(f"Processing RAG query with k={k_to_use}: '{query[:100]}...'")
+
+        original_k = self.retriever.k
+        if k_to_use != original_k:
+            self.retriever.k = k_to_use
+        
+        try:
+            retrieved_docs = self.retriever.get_relevant_documents(query)
+            if not retrieved_docs:
+                return {"answer": "The provided bibliography does not contain specific information on this topic.", "source": "no_docs_found", "cited_source_details": []}
+
+            context_str = self.format_docs(retrieved_docs)
+            
+            # MODIFIED: Added full logging as per user request
+            print(f"\n--- RAG INVOKE ---")
+            print(f"QUESTION: {query}")
+            print(f"CONTEXT:\n{context_str}")
+            
+            llm_answer = self.rag_chain.invoke(query, config={"context": context_str})
+
+            print(f"LLM_ANSWER: {llm_answer}")
+            print(f"--------------------\n")
+
+            structured_sources = [{
+                "source_document_name": doc.metadata.get('source_document_name', 'Unknown'),
+                "chunk_index": doc.metadata.get('chunk_index', 'N/A'),
+                "retrieval_score": doc.metadata.get("retrieval_score"),
+            } for doc in retrieved_docs]
+
+            if "the provided bibliography does not contain specific information" in llm_answer.lower():
+                 final_answer = llm_answer
+                 source_tag = "no_answer_in_bibliography"
+            else:
+                 final_answer = f"{llm_answer}\n\n*Source: Bibliography-Based*"
+                 source_tag = "bibliography_based"
+
+            return {
+                "query": query,
+                "answer": final_answer.strip(),
+                "source": source_tag,
+                "cited_source_details": structured_sources,
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error during RAG query processing: {e}", exc_info=True)
+            return {"answer": "An error occurred while processing your request.", "source": "system_error", "cited_source_details": []}
+        finally:
+            if k_to_use != original_k:
+                self.retriever.k = original_k
+    
+    def stream(self, query: str, top_k: Optional[int] = None) -> Iterator[str]:
+        if not self.rag_chain:
+            self.logger.error("RAG system not fully initialized for streaming.")
+            yield "Error: RAG system is not ready."
+            return
+
+        k_to_use = top_k if top_k is not None and top_k > 0 else self.retriever.k
+        self.logger.info(f"Processing RAG stream with k={k_to_use}: '{query[:100]}...'")
+
+        original_k = self.retriever.k
+        if k_to_use != original_k:
+            self.retriever.k = k_to_use
+        
+        try:
+            # Check for docs first to avoid streaming "no info" message
+            retrieved_docs = self.retriever.get_relevant_documents(query)
+            if not retrieved_docs:
+                yield "The provided bibliography does not contain specific information on this topic."
+                return
+            
+            # MODIFIED: Added full logging for streaming as per user request
+            context_str = self.format_docs(retrieved_docs)
+            print(f"\n--- RAG STREAM ---")
+            print(f"QUESTION: {query}")
+            print(f"CONTEXT:\n{context_str}")
+            print(f"STREAMING LLM_ANSWER...")
+            print(f"--------------------\n")
+                
+            yield from self.rag_chain.stream(query, config={"context": context_str})
+        except Exception as e:
+            self.logger.error(f"Error during RAG stream processing: {e}", exc_info=True)
+            yield "An error occurred while processing your request."
+        finally:
+            if k_to_use != original_k:
+                self.retriever.k = original_k
+
+
+# --- Groq Fallback Bot ---
+class GroqBot:
+    def __init__(self):
+        self.logger = logging.getLogger(__name__ + ".GroqBot")
+        if not GROQ_API_KEY:
+            self.client = None
+            self.logger.critical("GroqBot not initialized: BOT_API_KEY is missing.")
+            return
+        try:
+            self.client = LlamaIndexGroqClient(model=FALLBACK_LLM_MODEL_NAME, api_key=GROQ_API_KEY)
+            self.system_prompt = FALLBACK_SYSTEM_PROMPT
+        except Exception as e:
+            self.logger.error(f"Failed to initialize LlamaIndexGroqClient for Fallback Bot: {e}", exc_info=True)
+            self.client = None
+
+    def stream_response(self, context: dict) -> Iterator[str]:
+        if not self.client:
+            yield "The system is currently unable to process this request."
+            return
+
+        current_query = context.get('current_query', '')
+        chat_history = context.get('chat_history', [])
+        qa_info = context.get('qa_related_info', '')
+
+        messages = [ChatMessage(role="system", content=self.system_prompt)]
+        if chat_history:
+            messages.extend([ChatMessage(**msg) for msg in chat_history])
+        if qa_info:
+            messages.append(ChatMessage(role="system", content=f"**Potentially Relevant Q&A Information from other sources:**\n{qa_info}"))
+        messages.append(ChatMessage(role="user", content=f"**Current User Query:**\n{current_query}"))
+        
+        # MODIFIED: Added full logging as per user request
+        # The conversion to dict is necessary because ChatMessage is not directly JSON serializable
+        messages_for_print = [msg.dict() for msg in messages]
+        print(f"\n--- FALLBACK STREAM ---")
+        print(f"MESSAGES SENT TO LLM:\n{json.dumps(messages_for_print, indent=2)}")
+        print(f"STREAMING LLM_ANSWER...")
+        print(f"-----------------------\n")
+
+        try:
+            response_stream = self.client.stream_chat(messages)
+            for r_chunk in response_stream:
+                yield r_chunk.delta
+        except Exception as e:
+            self.logger.error(f"Groq API error in get_response (Fallback): {e}", exc_info=True)
+            yield "I am currently unable to process this request due to a technical issue."
+
+# ADDED: New function for formatting QA answers
+def get_answer_from_context(question: str, context: str, system_prompt: str) -> str:
+    """
+    Calls the LLM with a specific question and context from a QA source (CSV/XLSX).
+    """
+    logger.info(f"Formatting answer for question '{question[:50]}...' using QA context.")
+    try:
+        # Use the auxiliary model for this task for speed and cost-efficiency
+        formatter_llm = ChatGroq(
+            temperature=0.1,
+            groq_api_key=GROQ_API_KEY,
+            model_name=AUXILIARY_LLM_MODEL_NAME
+        )
+        
+        prompt_template = ChatPromptTemplate.from_template(system_prompt)
+        
+        chain = prompt_template | formatter_llm | StrOutputParser()
+        
+        # MODIFIED: Added full logging as per user request
+        print(f"\n--- QA FORMATTER ---")
+        print(f"QUESTION: {question}")
+        print(f"CONTEXT:\n{context}")
+
+        response = chain.invoke({
+            "context": context,
+            "question": question
+        })
+
+        print(f"LLM_ANSWER: {response}")
+        print(f"--------------------\n")
+        
+        return response.strip()
+        
+    except Exception as e:
+        logger.error(f"Error in get_answer_from_context: {e}", exc_info=True)
+        return "Sorry, I was unable to formulate an answer based on the available information."
+
+
+# --- Initialization and Interface Functions ---
+def get_id_from_gdrive_input(url_or_id: str) -> Optional[str]:
+    if not url_or_id: return None
+    patterns = [r"/folders/([a-zA-Z0-9_-]+)", r"/d/([a-zA-Z0-9_-]+)", r"id=([a-zA-Z0-9_-]+)"]
+    for pattern in patterns:
+        match = re.search(pattern, url_or_id)
+        if match: return match.group(1)
+    if "/" not in url_or_id and "=" not in url_or_id and len(url_or_id) > 15:
+        return url_or_id
+    return None
+
+def download_and_unzip_gdrive_folder(folder_id_or_url: str, target_dir: str) -> bool:
+    folder_id = get_id_from_gdrive_input(folder_id_or_url)
+    if not folder_id:
+        logger.error(f"Invalid Google Drive Folder ID or URL: {folder_id_or_url}")
+        return False
+    
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            logger.info(f"Attempting to download GDrive folder ID: {folder_id}")
+            download_path = gdown.download_folder(id=folder_id, output=temp_dir, quiet=False, use_cookies=False)
+            if not download_path or not os.listdir(temp_dir):
+                logger.error("gdown failed to download or extract the folder.")
+                return False
+
+            source_content_root = temp_dir
+            items_in_temp = os.listdir(temp_dir)
+            if len(items_in_temp) == 1 and os.path.isdir(os.path.join(temp_dir, items_in_temp[0])):
+                source_content_root = os.path.join(temp_dir, items_in_temp[0])
+            
+            logger.info(f"Moving contents from {source_content_root} to {target_dir}")
+            if os.path.exists(target_dir):
+                shutil.rmtree(target_dir)
+            shutil.copytree(source_content_root, target_dir)
+            logger.info(f"Successfully moved GDrive contents to {target_dir}")
+            return True
+        except Exception as e:
+            # MODIFIED: Corrected self.logger to logger
+            logger.error(f"Error during GDrive download/processing: {e}", exc_info=True)
+            return False
+
+def initialize_and_get_rag_system(force_rebuild: bool = False) -> Optional[KnowledgeRAG]:
+    if not GROQ_API_KEY:
+        logger.error("RAG system cannot be initialized without BOT_API_KEY.")
+        return None
+
+    if GDRIVE_SOURCES_ENABLED and GDRIVE_FOLDER_ID_OR_URL:
+        logger.info("Google Drive sources enabled. Downloading...")
+        if os.path.isdir(RAG_SOURCES_DIR):
+            logger.info(f"Clearing existing RAG sources directory: {RAG_SOURCES_DIR}")
+            shutil.rmtree(RAG_SOURCES_DIR)
+            os.makedirs(RAG_SOURCES_DIR)
+        
+        download_successful = download_and_unzip_gdrive_folder(GDRIVE_FOLDER_ID_OR_URL, RAG_SOURCES_DIR)
+        if not download_successful:
+            logger.error("Failed to download sources from Google Drive. Using local files if available.")
+
+    faiss_index_path = os.path.join(RAG_STORAGE_PARENT_DIR, RAG_FAISS_INDEX_SUBDIR_NAME)
+    if force_rebuild and os.path.exists(RAG_STORAGE_PARENT_DIR):
+        logger.info(f"Force Rebuild: Deleting existing index storage directory at '{RAG_STORAGE_PARENT_DIR}'")
+        shutil.rmtree(RAG_STORAGE_PARENT_DIR)
+        os.makedirs(RAG_STORAGE_PARENT_DIR)
+
+    try:
+        rag_instance = KnowledgeRAG(
+            index_storage_dir=RAG_STORAGE_PARENT_DIR,
+            embedding_model_name=RAG_EMBEDDING_MODEL_NAME,
+            groq_model_name_for_rag=RAG_LLM_MODEL_NAME,
+            use_gpu_for_embeddings=RAG_EMBEDDING_USE_GPU,
+            groq_api_key_for_rag=GROQ_API_KEY,
+            temperature=RAG_LLM_TEMPERATURE,
+        )
+        
+        should_build = True
+        if RAG_LOAD_INDEX_ON_STARTUP and not force_rebuild:
+            try:
+                rag_instance.load_index_from_disk(k=RAG_DEFAULT_RETRIEVER_K)
+                logger.info("RAG index loaded successfully from disk.")
+                should_build = False
+            except FileNotFoundError:
+                logger.warning("Index not found on disk. Will attempt to build.")
+            except Exception as e:
+                logger.error(f"Error loading index: {e}. Will attempt to rebuild.", exc_info=True)
+
+        if should_build:
+            logger.info("Building new RAG index from source data...")
+            rag_instance.build_index_from_source_files(
+                source_folder_path=RAG_SOURCES_DIR,
+                k=RAG_DEFAULT_RETRIEVER_K
+            )
+        
+        return rag_instance
+
+    except Exception as e:
+        logger.critical(f"FATAL: Failed to initialize RAG system: {e}", exc_info=True)
+        return None
+
+groq_bot_instance = GroqBot()
+
+# ADDED: New function to handle auxiliary model calls (This function is no longer used, replaced by get_answer_from_context)
+def get_auxiliary_chat_response(messages: List[Dict]) -> str:
+    """
+    Handles requests for auxiliary tasks like generating titles or follow-up questions.
+    Uses a separate, smaller model for efficiency.
+    """
+    logger.info(f"Routing auxiliary request to model: {AUXILIARY_LLM_MODEL_NAME}")
+    try:
+        # Initialize a dedicated client for this call to use the specific auxiliary model
+        aux_client = ChatGroq(
+            temperature=0.2, # A bit more creative than RAG, but still grounded
+            groq_api_key=GROQ_API_KEY,
+            model_name=AUXILIARY_LLM_MODEL_NAME
+        )
+        response = aux_client.invoke(messages)
+        return response.content
+    except Exception as e:
+        logger.error(f"Error with auxiliary model call: {e}", exc_info=True)
+        return "Could not generate suggestions."

requirements.txt

@@ -1 +1,30 @@
-huggingface_hub==0.25.2
+Flask==3.0.3
+Flask_Cors==5.0.0
+numpy
+pandas==2.2.3
+#rapidfuzz==3.10.1
+Requests==2.32.3
+#scikit_learn==1.4.1.post1
+#scikit_learn==1.5.2
+psycopg2-binary==2.9.10
+python-dotenv==1.0.1
+apscheduler==3.11.0
+redis==3.5.3
+faiss-cpu==1.10.0
+groq==0.15.0
+llama_index==0.12.13
+llama_index.llms.groq==0.3.1
+#langchain_groq==0.2.4
+#langchain_core==0.3.39
+sentence_transformers==3.4.0
+gunicorn
+llama-index-embeddings-huggingface==0.5.4
+onnxruntime==1.22.0
+langchain-groq==0.3.2
+python-docx==1.1.2
+langchain_community==0.3.23
+requests==2.32.3
+gdown==5.2.0
+pymupdf==1.25.5
+openpyxl==3.1.5
+# must install https://aka.ms/vs/17/release/vc_redist.x64.exe

sources/Endodontics%20appendix%204.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8b972d927a7a01f466cfe30e24834e653009926891919df1078f3563b718d1b
+size 188535

sources/Endodontics%20book.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:045c84eef3be88136f67bea03299a5e54ff121a017b8d6575d2fa7dd1269d460
+size 19170850

sources/Endodontics%20cap%201.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2956cff53ab6755bd5769d96ea6e021d21992d66898fa7ad19d59a651db08552
+size 1393891

sources/Endodontics%20cap%2010.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d60888b81a8ba43e14a85dee20a485120c6a168e75d8cb44f51f98ea1dc86c6
+size 858903

sources/Endodontics%20cap%2011.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cdfa6604cc9bdad78c8ffb4ec61e6ae5f12c3b75f0b52d45bab57c43983e676
+size 1132015

sources/Endodontics%20cap%2012.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b42e9dca8977115f11342fd3d264576c5b7224a51b82f3646d3a766669417f91
+size 1539100

sources/Endodontics%20cap%2013.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed0b9c196e3afe7477709996a261580721c42c391ecab6a0183ef4261b5a86fd
+size 1625649

sources/Endodontics%20cap%2014.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92e0c462d8eb85d94b08ba84ad2270c86e87a678dea4be0d700f910af89c64e7
+size 1357984

sources/Endodontics%20cap%2015.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54d2a579eed28d2bb58c8aca669dc6baaad1824a156517212494b0557f4837d7
+size 944072

sources/Endodontics%20cap%2016.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ab55472e1887827d5f55e51993ebb5ab9a031fafb6e81db258b82277c7ce4e5
+size 681335

sources/Endodontics%20cap%2017.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f5c395668f607f37c3f67131bdb0f82bc792b3bdbabb03071abd623d066db27
+size 528079

sources/Endodontics%20cap%2018.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8cb86f5fa0e481b2d91db3e93dcabe580a447630f7716d0f8309d5a018bbc64
+size 746444

sources/Endodontics%20cap%2019.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5746dc123bc59f7373699d48e753ad05552acaa18c743dc5026990e603686b2
+size 830885

sources/Endodontics%20cap%202.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b06e9d058a6f05afb04bfef398b8d0f36c02e4d78aba873dc294652c9fe2517c
+size 612225

sources/Endodontics%20cap%2020.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:127d6f0efcac6fc7d12cbee72f9f0cbf05136be860e5837662037d267e8c621b
+size 609023

sources/Endodontics%20cap%2021.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3b2e5699b2a92f2bb5187e825aa5f11afc1d2ed54bce56e6ba56fab699f2a4
+size 464443

sources/Endodontics%20cap%2022.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9dee2fd5d783cdf4f53b8e3d79ab012369e1c55ade79c4e5ee7544c83d690f
+size 561329

sources/Endodontics%20cap%203.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a79808a3da34f9817136e441254449dc5504f115cae1190505b8ce5f3c3f9b51
+size 786616

sources/Endodontics%20cap%204.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3223d4b6c6ae9b1b66bd5f0ac750bf8e3a514e7941555632ad98e67202929993
+size 810838

sources/Endodontics%20cap%205.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe63ba988c8c88207a7482c677983cd5f1a0b06cb5bc210b650a657b20924ebf
+size 761083

sources/Endodontics%20cap%206.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c490f38f73cd72e8bde876bb082964d43d1080abd1593ab7c4e53ca97a4f492
+size 1056526

sources/Endodontics%20cap%207.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b1f2f03a024f0520a798a7e7bd2c128cc5eda3d929a6b56d9a2b9dde184d14
+size 1619923

sources/Endodontics%20cap%208.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:159d99387578b0d9164e243ae27f7ee77ee998aa8c71dc15310e6ca274d96f7e
+size 751157

sources/Endodontics%20cap%209.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db47bb570fa521c1e7462e07bd22f9ce38485c2b262b0c774d16ba7795844179
+size 498083

sources/database.csv

@@ -0,0 +1 @@
+Question,Answer,Image

sources/general_qa.csv

@@ -0,0 +1 @@
+Question,Answer,Image

sources/greetings.csv

@@ -0,0 +1 @@
+Question,Answer,Image

sources/personal_qa.csv

@@ -0,0 +1 @@
+Question,Answer,Image

sources/preguntas chatbot_01.xlsx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65e4dfe9f1e7dad838718a016a486ca8a9e3c99068fcc22ae6e2e2064fd86943
+size 156246

system_prompts.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+This module centralizes all system prompts for the specialized dental chatbot application.
+This allows for easy management and updating of prompts without altering core logic.
+"""
+
+# --- RAG System Prompt for Bibliography-Based Answers ---
+# This prompt instructs the LLM to answer based *only* on the context provided
+# by the RAG system from scientific documents (PDFs, etc.).
+# Placeholders {context} and {question} will be filled by the LangChain pipeline.
+RAG_SYSTEM_PROMPT = """You are a specialized dental assistant AI. Your role is to provide accurate, evidence-based information on a specific dental topic.
+
+**Your Task:**
+Your primary task is to answer the user's question accurately and concisely, based *exclusively* on the "Provided Document Excerpts" below. These excerpts are from vetted scientific and dental publications.
+
+**Provided Document Excerpts:**
+{context}
+
+**User Question:**
+{question}
+
+---
+**Core Instructions:**
+1. **Language:** Your default language is **Spanish**. But follow the language of user. If they ask question in Spanish, reply in Spanish. If they ask in English, reply in English, even if the context is Spanish.
+2. **Strictly Adhere to Context:** Your answer **must** be derived solely from the "Provided Document Excerpts." Do not use any external knowledge or make assumptions beyond what is presented in the text.
+3.  **Professional Tone:** Maintain a clinical, objective, and professional tone suitable for a dental context.
+4.  **Do Not Speculate:** If the provided excerpts do not contain the information needed to answer the question, you must not invent an answer.
+5.  **Handling Unanswerable Questions:** If you cannot answer the question based on the provided excerpts, respond with: "The provided bibliography does not contain specific information on this topic." Do not attempt to guide the user elsewhere or apologize.
+6.  **No Self-Reference:** Do not mention that you are an AI, that you are "looking at documents," or refer to the "provided excerpts" in your final answer. Simply present the information as requested.
+
+**Answer Format:**
+Provide a direct answer to the user's question based on the information available.
+
+**Answer:**"""
+
+
+# --- Fallback System Prompt for General/Triage Purposes ---
+# REVISED: This prompt is now much stricter and will only handle dental-related queries.
+FALLBACK_SYSTEM_PROMPT = """You are a specialized dental assistant AI. Your one and only role is to answer questions strictly related to dentistry.
+
+**Core Instructions:**
+1.  **Dental Focus Only:** You MUST NOT engage in any general conversation, small talk, or answer questions outside the scope of dentistry.
+2.  **Handle Out-of-Scope Questions:** If the user's question is unrelated to dentistry, you must respond with the following exact phrase: "I am a dental assistant AI and my capabilities are limited to dental topics. Do you have a question about oral health?"
+3.  **Stateful Conversation:** Pay attention to the `Prior Conversation History` to understand the context of the user's dental inquiries.
+4.  **Professional Tone:** Always be polite, helpful, and professional.
+5.  **Do Not Make Up Clinical Advice:** Do not provide medical diagnoses or treatment plans. You can provide general information but should always recommend consulting a professional for personal health concerns.
+
+**Response Guidance:**
+- Review the `Prior Conversation History` to understand the context.
+- Formulate a helpful, professional answer to the `Current User Query` if it is about dentistry.
+"""
+
+# ADDED: New prompt to format answers based on structured data from CSV/XLSX files.
+QA_FORMATTER_PROMPT = """You are a helpful assistant. You will be given a user's question and structured data from a database row that is highly relevant to the question.
+Your task is to formulate a natural, conversational answer to the user's question based *only* on the provided data.
+
+- Synthesize the information from the data fields into a coherent response.
+- Do not just list the data. Create a proper sentence or paragraph.
+- If the data contains a 'Fuente' or 'Source' field, cite it at the end of your answer like this: (Source: [source_value]).
+
+**Provided Data:**
+{context}
+
+**User Question:**
+{question}
+
+**Answer:**"""