Upload 10 files

Dockerfile

@@ -0,0 +1,35 @@
+# Use an official Python runtime as a parent image
+FROM python:3.10-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the requirements file
+COPY requirements.txt requirements.txt
+
+# Install Python packages
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . /app
+
+# Create a non-root user
+RUN useradd -m -u 1000 user
+
+# Change ownership
+RUN chown -R user:user /app
+
+# Switch to the non-root user
+USER user
+
+# Expose the port Gunicorn will run on (Using 7860 as in CMD)
+EXPOSE 7860
+
+# Command to run the app
+CMD ["python", "app.py", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,1023 @@
+from flask import Flask, request, send_file, abort, jsonify, url_for, render_template, Response
+from flask_cors import CORS
+import pandas as pd
+from sentence_transformers import SentenceTransformer, util
+import torch
+from dataclasses import dataclass
+from typing import List, Dict, Tuple, Optional, Any, Iterator
+from collections import deque
+import os
+import logging
+import atexit
+from threading import Thread, Lock
+import time
+from datetime import datetime
+from uuid import uuid4 as generate_uuid
+import csv as csv_lib
+import functools
+import json
+import re
+import subprocess
+import sys
+import sqlite3
+import io
+
+from dotenv import load_dotenv
+
+# Load environment variables from .env file AT THE VERY TOP
+load_dotenv()
+
+# Import RAG system and Fallback LLM from llm_handling AFTER load_dotenv
+from llm_handling import (
+    initialize_and_get_rag_system,
+    KnowledgeRAG,
+    groq_bot_instance,
+    RAG_SOURCES_DIR,
+    RAG_STORAGE_PARENT_DIR,
+    RAG_CHUNKED_SOURCES_FILENAME,
+    get_answer_from_context,
+    stream_answer_from_context # <-- ADDED IMPORT
+)
+from system_prompts import QA_FORMATTER_PROMPT
+
+
+# Setup logging (remains global for the app)
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler("app_hybrid_rag.log"),
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__) # Main app logger
+
+# --- Application Constants and Configuration ---
+ADMIN_USERNAME = os.getenv('FLASK_ADMIN_USERNAME', 'admin')
+ADMIN_PASSWORD = os.getenv('FLASK_ADMIN_PASSWORD', 'admin')
+FLASK_APP_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
+FLASK_APP_PORT = int(os.getenv("FLASK_PORT", "7860"))
+FLASK_DEBUG_MODE = os.getenv("FLASK_DEBUG", "True").lower() == "true"
+_APP_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+TEXT_EXTRACTIONS_DIR = os.path.join(_APP_BASE_DIR, 'text_extractions')
+RELATED_QUESTIONS_TO_SHOW = 10
+QUESTIONS_TO_SEND_TO_GROQ_QA = 3
+# MODIFIED: Lowered the threshold to make it easier to trigger the Excel/CSV logic
+LLM_FORMATTER_CONFIDENCE_THRESHOLD = int(os.getenv("LLM_FORMATTER_CONFIDENCE_THRESHOLD", "65"))
+HIGH_CONFIDENCE_THRESHOLD = 90 # For greetings, which are answered directly.
+CHAT_HISTORY_TO_SEND = int(os.getenv("CHAT_HISTORY_TO_SEND", "5"))
+CHAT_LOG_FILE = os.path.join(_APP_BASE_DIR, 'chat_history.csv')
+
+rag_system: Optional[KnowledgeRAG] = None
+
+# --- Persistent Chat History Management using SQLite ---
+class ChatHistoryManager:
+    def __init__(self, db_path):
+        self.db_path = db_path
+        self.lock = Lock()
+        self._create_table()
+        logger.info(f"SQLite chat history manager initialized at: {self.db_path}")
+
+    def _get_connection(self):
+        conn = sqlite3.connect(self.db_path, timeout=10)
+        return conn
+
+    def _create_table(self):
+        with self.lock:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS chat_histories (
+                        session_id TEXT PRIMARY KEY,
+                        history TEXT NOT NULL
+                    )
+                """)
+                conn.commit()
+
+    def get_history(self, session_id: str, limit_turns: int = 5) -> list:
+        try:
+            with self._get_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT history FROM chat_histories WHERE session_id = ?", (session_id,))
+                row = cursor.fetchone()
+                if row:
+                    history_list = json.loads(row[0])
+                    return history_list[-(limit_turns * 2):]
+                else:
+                    return []
+        except Exception as e:
+            logger.error(f"Error fetching history for session {session_id}: {e}", exc_info=True)
+            return []
+
+    def update_history(self, session_id: str, query: str, answer: str):
+        with self.lock:
+            try:
+                with self._get_connection() as conn:
+                    cursor = conn.cursor()
+                    cursor.execute("SELECT history FROM chat_histories WHERE session_id = ?", (session_id,))
+                    row = cursor.fetchone()
+                    
+                    history = json.loads(row[0]) if row else []
+                    
+                    history.append({'role': 'user', 'content': query})
+                    history.append({'role': 'assistant', 'content': answer})
+
+                    updated_history_json = json.dumps(history)
+
+                    cursor.execute("""
+                        INSERT OR REPLACE INTO chat_histories (session_id, history)
+                        VALUES (?, ?)
+                    """, (session_id, updated_history_json))
+                    conn.commit()
+            except Exception as e:
+                logger.error(f"Error updating history for session {session_id}: {e}", exc_info=True)
+                
+    def clear_history(self, session_id: str):
+        with self.lock:
+            try:
+                with self._get_connection() as conn:
+                    cursor = conn.cursor()
+                    cursor.execute("""
+                        INSERT OR REPLACE INTO chat_histories (session_id, history)
+                        VALUES (?, ?)
+                    """, (session_id, json.dumps([])))
+                    conn.commit()
+                logger.info(f"Chat history cleared for session: {session_id}")
+            except Exception as e:
+                 logger.error(f"Error clearing history for session {session_id}: {e}", exc_info=True)
+
+
+# --- EmbeddingManager for CSV QA (remains in app.py) ---
+@dataclass
+class QAEmbeddings:
+    questions: List[str]
+    question_map: List[int]
+    embeddings: torch.Tensor
+    df_qa: pd.DataFrame
+    original_questions: List[str]
+
+class EmbeddingManager:
+    def __init__(self, model_name='all-MiniLM-L6-v2'):
+        self.model = SentenceTransformer(model_name)
+        self.embeddings = {
+            'general': None,
+            'personal': None,
+            'greetings': None
+        }
+        logger.info(f"EmbeddingManager initialized with model: {model_name}")
+
+    def _process_questions(self, df: pd.DataFrame) -> Tuple[List[str], List[int], List[str]]:
+        questions = []
+        question_map = []
+        original_questions = []
+
+        if 'Question' not in df.columns:
+            logger.warning(f"DataFrame for EmbeddingManager is missing 'Question' column. Cannot process questions from it.")
+            return questions, question_map, original_questions
+
+        for idx, question_text_raw in enumerate(df['Question']):
+            if pd.isna(question_text_raw):
+                continue
+            question_text_cleaned = str(question_text_raw).strip()
+            if not question_text_cleaned or question_text_cleaned.lower() == "nan":
+                continue
+
+            questions.append(question_text_cleaned)
+            question_map.append(idx)
+            original_questions.append(question_text_cleaned)
+
+        return questions, question_map, original_questions
+
+    def update_embeddings(self, general_qa: pd.DataFrame, personal_qa: pd.DataFrame, greetings_qa: pd.DataFrame):
+        gen_questions, gen_question_map, gen_original_questions = self._process_questions(general_qa)
+        gen_embeddings = self.model.encode(gen_questions, convert_to_tensor=True, show_progress_bar=False) if gen_questions else None
+
+        pers_questions, pers_question_map, pers_original_questions = self._process_questions(personal_qa)
+        pers_embeddings = self.model.encode(pers_questions, convert_to_tensor=True, show_progress_bar=False) if pers_questions else None
+
+        greet_questions, greet_question_map, greet_original_questions = self._process_questions(greetings_qa)
+        greet_embeddings = self.model.encode(greet_questions, convert_to_tensor=True, show_progress_bar=False) if greet_questions else None
+
+        self.embeddings['general'] = QAEmbeddings(
+            questions=gen_questions, question_map=gen_question_map, embeddings=gen_embeddings,
+            df_qa=general_qa, original_questions=gen_original_questions
+        )
+        self.embeddings['personal'] = QAEmbeddings(
+            questions=pers_questions, question_map=pers_question_map, embeddings=pers_embeddings,
+            df_qa=personal_qa, original_questions=pers_original_questions
+        )
+        self.embeddings['greetings'] = QAEmbeddings(
+            questions=greet_questions, question_map=greet_question_map, embeddings=greet_embeddings,
+            df_qa=greetings_qa, original_questions=greet_original_questions
+        )
+        logger.info("CSV QA embeddings updated in EmbeddingManager.")
+
+    def find_best_answers(self, user_query: str, qa_type: str, top_n: int = 5) -> Tuple[List[float], List[str], List[str], List[str], List[int]]:
+        qa_data = self.embeddings[qa_type]
+        if qa_data is None or qa_data.embeddings is None or len(qa_data.embeddings) == 0:
+            return [], [], [], [], []
+
+        query_embedding_tensor = self.model.encode([user_query], convert_to_tensor=True, show_progress_bar=False)
+        if not isinstance(qa_data.embeddings, torch.Tensor):
+                qa_data.embeddings = torch.tensor(qa_data.embeddings) # Safeguard
+
+        cos_scores = util.cos_sim(query_embedding_tensor, qa_data.embeddings)[0]
+
+        top_k = min(top_n, len(cos_scores))
+        if top_k == 0:
+            return [], [], [], [], []
+
+        top_scores_tensor, indices_tensor = torch.topk(cos_scores, k=top_k)
+
+        top_confidences = [score.item() * 100 for score in top_scores_tensor]
+        top_indices_mapped = []
+        top_questions = []
+
+        for idx_tensor in indices_tensor:
+            item_idx = idx_tensor.item()
+            if item_idx < len(qa_data.question_map) and item_idx < len(qa_data.original_questions):
+                original_df_idx = qa_data.question_map[item_idx]
+                if original_df_idx < len(qa_data.df_qa):
+                    top_indices_mapped.append(original_df_idx)
+                    top_questions.append(qa_data.original_questions[item_idx])
+                else:
+                    logger.warning(f"Index out of bounds: original_df_idx {original_df_idx} for df_qa length {len(qa_data.df_qa)}")
+            else:
+                logger.warning(f"Index out of bounds: item_idx {item_idx} for question_map/original_questions")
+
+        valid_count = len(top_indices_mapped)
+        top_confidences = top_confidences[:valid_count]
+        top_questions = top_questions[:valid_count]
+        
+        if 'Respuesta' in qa_data.df_qa.columns:
+            answer_col = 'Respuesta'
+        elif 'Answer' in qa_data.df_qa.columns:
+            answer_col = 'Answer'
+        else:
+            answer_col = None
+
+        if answer_col:
+            top_answers = [str(qa_data.df_qa[answer_col].iloc[i]) for i in top_indices_mapped]
+        else:
+            top_answers = [str(qa_data.df_qa['Question'].iloc[i]) for i in top_indices_mapped]
+
+        top_images = [str(qa_data.df_qa['Image'].iloc[i]) if 'Image' in qa_data.df_qa.columns and pd.notna(qa_data.df_qa['Image'].iloc[i]) else None for i in top_indices_mapped]
+
+        return top_confidences, top_questions, top_answers, top_images, top_indices_mapped
+
+# --- DatabaseMonitor for personal_qa.csv placeholders (remains in app.py) ---
+class DatabaseMonitor:
+    def __init__(self, database_path):
+        self.logger = logging.getLogger(__name__ + ".DatabaseMonitor")
+        self.database_path = database_path
+        self.last_modified = None
+        self.last_size = None
+        self.df = None
+        self.lock = Lock()
+        self.running = True
+        self._load_database()
+        self.monitor_thread = Thread(target=self._monitor_database, daemon=True)
+        self.monitor_thread.start()
+        self.logger.info(f"DatabaseMonitor initialized for: {database_path}")
+
+    def _load_database(self):
+        try:
+            if not os.path.exists(self.database_path):
+                self.logger.warning(f"Personal data file not found: {self.database_path}.")
+                self.df = None
+                return
+            with self.lock:
+                self.df = pd.read_csv(self.database_path, encoding='cp1252')
+                self.last_modified = os.path.getmtime(self.database_path)
+                self.last_size = os.path.getsize(self.database_path)
+                self.logger.info(f"Personal data file reloaded: {self.database_path}")
+        except Exception as e:
+            self.logger.error(f"Error loading personal data file '{self.database_path}': {e}", exc_info=True)
+            self.df = None
+
+    def _monitor_database(self):
+        while self.running:
+            try:
+                if not os.path.exists(self.database_path):
+                    if self.df is not None:
+                        self.logger.warning(f"Personal data file disappeared: {self.database_path}")
+                        self.df = None; self.last_modified = None; self.last_size = None
+                    time.sleep(5)
+                    continue
+                current_modified = os.path.getmtime(self.database_path); current_size = os.path.getsize(self.database_path)
+                if (self.last_modified is None or current_modified != self.last_modified or
+                    self.last_size is None or current_size != self.last_size):
+                    self.logger.info("Personal data file change detected.")
+                    self._load_database()
+                time.sleep(1)
+            except Exception as e:
+                self.logger.error(f"Error monitoring personal data file: {e}", exc_info=True)
+                time.sleep(5)
+
+    def get_data(self, user_id):
+        with self.lock:
+            if self.df is not None and user_id:
+                try:
+                    if 'id' not in self.df.columns:
+                        self.logger.warning("'id' column not found in personal_data.csv")
+                        return None
+                    id_col_type = self.df['id'].dtype
+                    target_user_id = user_id
+                    if pd.api.types.is_numeric_dtype(id_col_type):
+                        try:
+                            if user_id is None: return None
+                            valid_ids = self.df['id'].dropna()
+                            if not valid_ids.empty:
+                                target_user_id = type(valid_ids.iloc[0])(user_id)
+                            else:
+                                target_user_id = int(user_id)
+                        except (ValueError, TypeError):
+                            self.logger.warning(f"Could not convert user_id '{user_id}' to numeric type {id_col_type}")
+                            return None
+                    user_data = self.df[self.df['id'] == target_user_id]
+                    if not user_data.empty: return user_data.iloc[0].to_dict()
+                except Exception as e:
+                    self.logger.error(f"Error retrieving data for user_id {user_id}: {e}", exc_info=True)
+            return None
+
+    def stop(self):
+        self.running = False
+        if hasattr(self, 'monitor_thread') and self.monitor_thread.is_alive():
+            self.monitor_thread.join(timeout=5)
+        self.logger.info("DatabaseMonitor stopped.")
+
+# --- Flask App Initialization ---
+app = Flask(__name__)
+CORS(app, resources={r"/*": {"origins": "*"}}, supports_credentials=True)
+
+# --- Initialize Managers ---
+embedding_manager = EmbeddingManager()
+history_manager = ChatHistoryManager('chat_history.db')
+database_csv_path = os.path.join(RAG_SOURCES_DIR, 'database.csv')
+personal_data_monitor = DatabaseMonitor(database_csv_path)
+
+# --- Helper Functions (App specific) ---
+def clean_html_from_text(text: str) -> str:
+    """Removes HTML tags from a string using a simple regex."""
+    if not isinstance(text, str):
+        return text
+    clean_text = re.sub(r'<[^>]+>', '', text)
+    return clean_text.strip()
+
+def normalize_text(text):
+    if isinstance(text, str):
+        replacements = {
+            '\x91': "'", '\x92': "'", '\x93': '"', '\x94': '"',
+            '\x96': '-', '\x97': '-', '\x85': '...', '\x95': '-',
+            '"': '"', '"': '"', '‘': "'", '’': "'",
+            '–': '-', '—': '-', '…': '...', '•': '-',
+        }
+        for old, new in replacements.items(): text = text.replace(old, new)
+    return text
+
+def require_admin_auth(f):
+    @functools.wraps(f)
+    def decorated(*args, **kwargs):
+        auth = request.authorization
+        if not auth or auth.username != ADMIN_USERNAME or auth.password != ADMIN_PASSWORD:
+            return Response('Admin auth failed.', 401, {'WWW-Authenticate': 'Basic realm="Admin Login Required"'})
+        return f(*args, **kwargs)
+    return decorated
+
+def initialize_chat_log():
+    if not os.path.exists(CHAT_LOG_FILE):
+        with open(CHAT_LOG_FILE, 'w', newline='', encoding='utf-8') as f:
+            writer = csv_lib.writer(f)
+            writer.writerow(['sl', 'date_time', 'session_id', 'user_id', 'query', 'answer'])
+
+def store_chat_history(sid: str, uid: Optional[str], query: str, resp: Dict[str, Any]):
+    try:
+        answer = str(resp.get('answer', ''))
+        history_manager.update_history(sid, query, answer)
+
+        initialize_chat_log()
+        next_sl = 1
+        try:
+            if os.path.exists(CHAT_LOG_FILE) and os.path.getsize(CHAT_LOG_FILE) > 0:
+                df_log = pd.read_csv(CHAT_LOG_FILE, on_bad_lines='skip')
+                if not df_log.empty and 'sl' in df_log.columns and pd.api.types.is_numeric_dtype(df_log['sl'].dropna()):
+                    if not df_log['sl'].dropna().empty:
+                        next_sl = int(df_log['sl'].dropna().max()) + 1
+        except Exception as e:
+            logger.error(f"Error reading SL from {CHAT_LOG_FILE}: {e}", exc_info=True)
+
+        with open(CHAT_LOG_FILE, 'a', newline='', encoding='utf-8') as f:
+            csv_lib.writer(f).writerow([next_sl, datetime.now().strftime('%Y-%m-%d %H:%M:%S'), sid, uid or "N/A", query, answer])
+
+    except Exception as e:
+        logger.error(f"Error in store_chat_history for session {sid}: {e}", exc_info=True)
+
+def get_formatted_chat_history(session_id: str) -> List[Dict[str, str]]:
+    if not session_id:
+        return []
+    return history_manager.get_history(session_id, limit_turns=CHAT_HISTORY_TO_SEND)
+
+def get_qa_context_for_groq(all_questions: List[Dict]) -> str:
+    valid_qa_pairs = []
+    non_greeting_questions = [q for q in all_questions if q.get('source_type') != 'greetings']
+    sorted_questions = sorted(non_greeting_questions, key=lambda x: x.get('confidence', 0), reverse=True)
+
+    for qa in sorted_questions[:QUESTIONS_TO_SEND_TO_GROQ_QA]:
+        answer = qa.get('answer')
+        if (not pd.isna(answer) and isinstance(answer, str) and answer.strip() and
+            "not available" not in answer.lower()):
+            valid_qa_pairs.append(f"Q: {qa.get('question')}\nA: {answer}")
+    return '\n'.join(valid_qa_pairs)
+
+def replace_placeholders_in_answer(answer, db_data):
+    if pd.isna(answer) or str(answer).strip() == '':
+        return "Sorry, this information is not available yet"
+    answer_str = str(answer)
+    placeholders = re.findall(r'\{(\w+)\}', answer_str)
+    if not placeholders: return answer_str
+    if db_data is None:
+        return "To get this specific information, please ensure you are logged in or have provided your user ID."
+    missing_count = 0; replacements_made = 0
+    for placeholder in set(placeholders):
+        key = placeholder.strip()
+        value = db_data.get(key)
+        if value is None or (isinstance(value, float) and pd.isna(value)) or str(value).strip() == '':
+            answer_str = answer_str.replace(f'{{{key}}}', "not available")
+            missing_count += 1
+        else:
+            answer_str = answer_str.replace(f'{{{key}}}', str(value))
+            replacements_made +=1
+    if missing_count == len(placeholders) and len(placeholders) > 0 :
+        return "Sorry, some specific details for you are not available at the moment."
+    if "not available" in answer_str.lower() and replacements_made < len(placeholders):
+         if answer_str == "not available" and len(placeholders) == 1:
+             return "Sorry, this information is not available yet."
+    if re.search(r'\{(\w+)\}', answer_str):
+        logger.warning(f"Unresolved placeholders remain after replacement attempt: {answer_str}")
+        answer_str = re.sub(r'\{(\w+)\}', "a specific detail", answer_str)
+        if "a specific detail" in answer_str and not "Sorry" in answer_str:
+            return "Sorry, I couldn't retrieve all the specific details for this answer. " + answer_str
+        return "Sorry, I couldn't retrieve all the specific details for this answer. Some information has been generalized."
+    return answer_str
+
+# --- Non-Streaming Logic ---
+def get_hybrid_response_logic_non_streaming(user_query: str, session_id: str, user_id: Optional[str], chat_history: Optional[List[Dict]] = None) -> Dict[str, Any]:
+    global rag_system
+
+    if not user_query: return {'error': 'No query provided'}
+    if not session_id: return {'error': 'session_id is required'}
+
+    personal_db_data = personal_data_monitor.get_data(user_id) if user_id else None
+
+    # Get candidates from QA files
+    conf_greet, q_greet, a_greet, img_greet, idx_greet = embedding_manager.find_best_answers(user_query, 'greetings', top_n=1)
+    conf_pers, q_pers, a_pers, img_pers, idx_pers = embedding_manager.find_best_answers(user_query, 'personal', top_n=5)
+    conf_gen, q_gen, a_gen, img_gen, idx_gen = embedding_manager.find_best_answers(user_query, 'general', top_n=5)
+
+    # Handle greetings separately with a high confidence check
+    if conf_greet and conf_greet[0] >= HIGH_CONFIDENCE_THRESHOLD:
+        response_data = {
+            'query': user_query, 'answer': a_greet[0], 
+            'confidence': conf_greet[0], 
+            'original_question': q_greet[0], 
+            'source': 'greetings_qa'
+        }
+        if img_greet and img_greet[0]:
+            response_data['image_url'] = url_for('static', filename=img_greet[0], _external=True)
+        store_chat_history(session_id, user_id, user_query, response_data)
+        return response_data
+
+    # --- MODIFIED LOGIC ---
+    # Combine general and personal candidates and send top 5 to LLM, regardless of confidence
+    all_qa_candidates = []
+    if conf_pers:
+        for c, q, a, img, idx in zip(conf_pers, q_pers, a_pers, img_pers, idx_pers):
+            processed_a = replace_placeholders_in_answer(a, personal_db_data)
+            if not ("Sorry, this information is not available yet" in processed_a or "To get this specific information" in processed_a):
+                all_qa_candidates.append({'question': q, 'answer': processed_a, 'image': img, 'confidence': c, 'source_type': 'personal', 'original_index': idx})
+    if conf_gen:
+        for c, q, a, img, idx in zip(conf_gen, q_gen, a_gen, img_gen, idx_gen):
+            if not (pd.isna(a) or str(a).strip() == '' or str(a).lower() == 'nan'):
+                all_qa_candidates.append({'question': str(a), 'answer': str(a), 'image': img, 'confidence': c, 'source_type': 'general', 'original_index': idx})
+
+    all_qa_candidates.sort(key=lambda x: x['confidence'], reverse=True)
+    top_5_candidates = all_qa_candidates[:5]
+
+    if top_5_candidates:
+        logger.info(f"Found {len(top_5_candidates)} relevant rows from CSV/XLSX. Sending to LLM for formatting.")
+        
+        context_chunks = []
+        for candidate in top_5_candidates:
+            source_type = candidate['source_type']
+            original_df = embedding_manager.embeddings[source_type].df_qa
+            matched_row_data = original_df.iloc[candidate['original_index']]
+            
+            row_dict = matched_row_data.to_dict()
+            row_context_str = "\n".join([f"- {key}: {value}" for key, value in row_dict.items() if pd.notna(value) and str(value).strip() != ''])
+            context_chunks.append(f"Matching Row {len(context_chunks) + 1} (From: {source_type} source file):\n{row_context_str}")
+        
+        full_context = "\n\n---\n\n".join(context_chunks)
+
+        final_answer = get_answer_from_context(
+            question=user_query,
+            context=full_context,
+            system_prompt=QA_FORMATTER_PROMPT
+        )
+        
+        response_data = {
+            'query': user_query, 
+            'answer': final_answer, 
+            'confidence': top_5_candidates[0]['confidence'],
+            'original_question': top_5_candidates[0]['question'], 
+            'source': 'xlsx_qa_llm_formatted'
+        }
+        if top_5_candidates[0].get('image'):
+            response_data['image_url'] = url_for('static', filename=top_5_candidates[0]['image'], _external=True)
+
+        related_questions_list = [{'question': c['question'], 'answer': c['answer'], 'match': c['confidence']} for c in all_qa_candidates[1:RELATED_QUESTIONS_TO_SHOW+1] if c['source_type'] != 'greetings']
+        response_data['related_questions'] = related_questions_list
+        
+        store_chat_history(session_id, user_id, user_query, response_data)
+        return response_data
+    # --- END OF MODIFIED LOGIC ---
+
+    # Fallback to RAG if no QA candidates were found
+    if rag_system and rag_system.retriever:
+        logger.info(f"Attempting FAISS RAG query for: {user_query[:50]}...")
+        rag_result = rag_system.invoke(user_query)
+        rag_answer = rag_result.get("answer")
+        
+        if rag_answer and "the provided bibliography does not contain specific information" not in rag_answer.lower():
+            logger.info(f"FAISS RAG system provided a valid answer: {rag_answer[:100]}...")
+            response_data = {
+                'query': user_query, 'answer': rag_answer, 'confidence': 85,
+                'source': 'document_rag_faiss', 'related_questions': [],
+                'document_sources_details': rag_result.get("cited_source_details")
+            }
+            store_chat_history(session_id, user_id, user_query, response_data)
+            return response_data
+
+    # Final fallback to general Groq model
+    logger.info(f"No high-confidence answer. Using Groq fallback.")
+    chat_history_messages_for_groq = chat_history if chat_history is not None else get_formatted_chat_history(session_id)
+    groq_context = {'current_query': user_query, 'chat_history': chat_history_messages_for_groq, 'qa_related_info': ""}
+    groq_stream = groq_bot_instance.stream_response(groq_context)
+    groq_answer = "".join([chunk for chunk in groq_stream])
+
+    response_data = {'query': user_query, 'answer': groq_answer, 'confidence': 75, 'source': 'groq_general_fallback', 'related_questions': []}
+    store_chat_history(session_id, user_id, user_query, response_data)
+    return response_data
+
+# --- Streaming Logic ---
+def generate_streaming_response(user_query: str, session_id: str, user_id: Optional[str], chat_history: Optional[List[Dict]] = None) -> Iterator[str]:
+    global rag_system
+    
+    personal_db_data = personal_data_monitor.get_data(user_id) if user_id else None
+    
+    # Get candidates from QA files
+    conf_greet, _, a_greet, _, _ = embedding_manager.find_best_answers(user_query, 'greetings', top_n=1)
+    conf_pers, q_pers, a_pers, img_pers, idx_pers = embedding_manager.find_best_answers(user_query, 'personal', top_n=5)
+    conf_gen, q_gen, a_gen, img_gen, idx_gen = embedding_manager.find_best_answers(user_query, 'general', top_n=5)
+
+    # Handle greetings separately
+    if conf_greet and conf_greet[0] >= HIGH_CONFIDENCE_THRESHOLD:
+        yield a_greet[0]
+        return
+
+    # --- MODIFIED LOGIC ---
+    # Combine general and personal candidates and stream a formatted response
+    all_qa_candidates = []
+    if conf_pers:
+        for c, q, a, img, idx in zip(conf_pers, q_pers, a_pers, img_pers, idx_pers):
+            processed_a = replace_placeholders_in_answer(a, personal_db_data)
+            if not ("Sorry, this information is not available yet" in processed_a or "To get this specific information" in processed_a):
+                all_qa_candidates.append({'question': q, 'answer': processed_a, 'image': img, 'confidence': c, 'source_type': 'personal', 'original_index': idx})
+    if conf_gen:
+        for c, q, a, img, idx in zip(conf_gen, q_gen, a_gen, img_gen, idx_gen):
+            if not (pd.isna(a) or str(a).strip() == '' or str(a).lower() == 'nan'):
+                all_qa_candidates.append({'question': str(a), 'answer': str(a), 'image': img, 'confidence': c, 'source_type': 'general', 'original_index': idx})
+
+    all_qa_candidates.sort(key=lambda x: x['confidence'], reverse=True)
+    top_5_candidates = all_qa_candidates[:5]
+
+    if top_5_candidates:
+        logger.info(f"Found {len(top_5_candidates)} relevant CSV/XLSX rows. Streaming formatted answer.")
+        
+        context_chunks = []
+        for candidate in top_5_candidates:
+            source_type = candidate['source_type']
+            original_df = embedding_manager.embeddings[source_type].df_qa
+            matched_row_data = original_df.iloc[candidate['original_index']]
+            
+            row_dict = matched_row_data.to_dict()
+            row_context_str = "\n".join([f"- {key}: {value}" for key, value in row_dict.items() if pd.notna(value) and str(value).strip() != ''])
+            context_chunks.append(f"Matching Row {len(context_chunks) + 1} (From: {source_type} source file):\n{row_context_str}")
+        
+        full_context = "\n\n---\n\n".join(context_chunks)
+
+        yield from stream_answer_from_context(
+            question=user_query,
+            context=full_context,
+            system_prompt=QA_FORMATTER_PROMPT
+        )
+        return
+    # --- END OF MODIFIED LOGIC ---
+
+    # Fallback to RAG if no QA candidates were found
+    if rag_system and rag_system.retriever:
+        logger.info(f"Attempting to stream from FAISS RAG for: {user_query[:50]}...")
+        rag_stream = rag_system.stream(user_query)
+        first_chunk = next(rag_stream, None)
+        
+        if first_chunk and "the provided bibliography does not contain specific information" not in first_chunk.lower():
+            logger.info("FAISS RAG streaming valid answer...")
+            yield first_chunk
+            yield from rag_stream
+            return
+    
+    # Final fallback to general Groq model
+    logger.info(f"No high-confidence CSV or RAG answer. Streaming from Groq fallback.")
+    chat_history_messages_for_groq = chat_history if chat_history is not None else get_formatted_chat_history(session_id)
+    groq_context = {'current_query': user_query, 'chat_history': chat_history_messages_for_groq, 'qa_related_info': ""}
+    yield from groq_bot_instance.stream_response(groq_context)
+
+def stream_formatter(logic_generator: Iterator[str], session_id: str, user_id: Optional[str], query: str) -> Iterator[str]:
+    chunk_id = f"chatcmpl-{str(generate_uuid())}"
+    model_name = "MedicalAssistantBot/v1"
+    full_response_chunks = []
+
+    for chunk in logic_generator:
+        if not chunk: continue
+        full_response_chunks.append(chunk)
+        response_json = {
+            "id": chunk_id, "object": "chat.completion.chunk", "created": int(time.time()), "model": model_name,
+            "choices": [{"index": 0, "delta": {"content": chunk}, "finish_reason": None}]
+        }
+        yield f"data: {json.dumps(response_json)}\n\n"
+
+    final_json = {
+        "id": chunk_id, "object": "chat.completion.chunk", "created": int(time.time()), "model": model_name,
+        "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+    }
+    yield f"data: {json.dumps(final_json)}\n\n"
+    yield "data: [DONE]\n\n"
+    
+    full_response = "".join(full_response_chunks)
+    
+    print(f"\n--- STREAMED FULL RESPONSE ---")
+    print(full_response)
+    print(f"------------------------------\n")
+
+    history_manager.update_history(session_id, query, full_response)
+
+# --- API Endpoints ---
+@app.route('/chat-bot', methods=['POST'])
+def get_answer_hybrid():
+    data = request.json
+    user_query = data.get('query', '')
+    user_query = clean_html_from_text(user_query)
+    user_id = data.get('user_id')
+    session_id = data.get('session_id')
+    
+    if not user_query or not session_id:
+        return jsonify({'error': 'query and session_id are required'}), 400
+
+    response_data = get_hybrid_response_logic_non_streaming(user_query, session_id, user_id, None)
+    return jsonify(response_data)
+
+@app.route('/v1/models', methods=['GET'])
+def list_models():
+    model_data = {
+        "object": "list",
+        "data": [{"id": "MedicalAssistantBot/v1", "object": "model", "created": int(time.time()), "owned_by": "user"}]
+    }
+    return jsonify(model_data)
+
+@app.route('/v1/chat/completions', methods=['POST'])
+def openai_compatible_chat_endpoint():
+    data = request.json
+    is_streaming = data.get("stream", False)
+    
+    messages = data.get("messages", [])
+    if not messages: return jsonify({"error": "No messages provided"}), 400
+        
+    user_query = messages[-1].get("content", "")
+    user_query = clean_html_from_text(user_query)
+    chat_history = messages[:-1]
+    session_id = data.get("conversation_id", f"webui-session-{str(generate_uuid())}")
+    user_id = None
+
+    if is_streaming:
+        logic_generator = generate_streaming_response(user_query, session_id, user_id, chat_history)
+        return Response(stream_formatter(logic_generator, session_id, user_id, user_query), mimetype='text/event-stream')
+    else:
+        full_response_dict = get_hybrid_response_logic_non_streaming(user_query, session_id, user_id, chat_history)
+        response_content = full_response_dict.get("answer", "Sorry, an error occurred.")
+        
+        openai_response = {
+            "id": f"chatcmpl-{str(generate_uuid())}", "object": "chat.completion", "created": int(time.time()),
+            "model": "MedicalAssistantBot/v1",
+            "choices": [{"index": 0, "message": {"role": "assistant", "content": response_content}, "finish_reason": "stop"}],
+            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
+        }
+        history_manager.update_history(session_id, user_query, response_content)
+        return jsonify(openai_response)
+
+
+# --- Admin and Utility Routes ---
+@app.route('/')
+def index_route():
+    template_to_render = 'chat-bot.html'
+    if not os.path.exists(os.path.join(app.root_path, 'templates', template_to_render)):
+        logger.warning(f"Template '{template_to_render}' not found. Serving basic message.")
+        return "Chatbot interface not found. Please ensure 'templates/chat-bot.html' exists.", 404
+    return render_template(template_to_render)
+
+@app.route('/admin/faiss_rag_status', methods=['GET'])
+@require_admin_auth
+def get_faiss_rag_status():
+    global rag_system
+    if not rag_system:
+        return jsonify({"error": "FAISS RAG system not initialized."}), 500
+    try:
+        status = {
+            "status": "Initialized" if rag_system.retriever else "Initialized (Retriever not ready)",
+            "index_storage_dir": rag_system.index_storage_dir,
+            "embedding_model": rag_system.embedding_model_name,
+            "groq_model": rag_system.groq_model_name,
+            "retriever_k": rag_system.retriever.k if rag_system.retriever else "N/A",
+            "processed_source_files": rag_system.processed_source_files,
+            "index_type": "FAISS",
+            "index_loaded_or_built": rag_system.vector_store is not None
+        }
+        if rag_system.vector_store and hasattr(rag_system.vector_store, 'index') and rag_system.vector_store.index:
+            try:
+                status["num_vectors_in_index"] = rag_system.vector_store.index.ntotal
+            except Exception:
+                status["num_vectors_in_index"] = "N/A (Could not get count)"
+        else:
+            status["num_vectors_in_index"] = "N/A (Vector store or index not available)"
+        return jsonify(status)
+    except Exception as e:
+        logger.error(f"Error getting FAISS RAG status: {e}", exc_info=True)
+        return jsonify({"error": str(e)}), 500
+
+@app.route('/admin/download_qa_database', methods=['GET'])
+@require_admin_auth
+def download_qa_database():
+    try:
+        output = io.BytesIO()
+        with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+            if embedding_manager.embeddings['general'] and embedding_manager.embeddings['general'].df_qa is not None:
+                embedding_manager.embeddings['general'].df_qa.to_excel(writer, sheet_name='General_QA', index=False)
+            
+            if embedding_manager.embeddings['personal'] and embedding_manager.embeddings['personal'].df_qa is not None:
+                embedding_manager.embeddings['personal'].df_qa.to_excel(writer, sheet_name='Personal_QA', index=False)
+
+            if embedding_manager.embeddings['greetings'] and embedding_manager.embeddings['greetings'].df_qa is not None:
+                embedding_manager.embeddings['greetings'].df_qa.to_excel(writer, sheet_name='Greetings', index=False)
+        
+        output.seek(0)
+        
+        return send_file(
+            output,
+            mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+            as_attachment=True,
+            download_name=f'qa_database_{datetime.now().strftime("%Y%m%d")}.xlsx'
+        )
+    except Exception as e:
+        logger.error(f"Error generating QA database file: {e}", exc_info=True)
+        return jsonify({'error': 'Failed to generate QA database file'}), 500
+
+@app.route('/admin/rebuild_faiss_index', methods=['POST'])
+@require_admin_auth
+def rebuild_faiss_index_route():
+    global rag_system
+    logger.info("Admin request to rebuild FAISS RAG index received. Starting two-step process.")
+
+    logger.info("Step 1: Running chunker.py to pre-process source documents.")
+    chunker_script_path = os.path.join(_APP_BASE_DIR, 'chunker.py')
+    chunked_json_output_path = os.path.join(RAG_STORAGE_PARENT_DIR, RAG_CHUNKED_SOURCES_FILENAME)
+
+    os.makedirs(TEXT_EXTRACTIONS_DIR, exist_ok=True)
+
+    if not os.path.exists(chunker_script_path):
+        logger.error(f"Chunker script not found at '{chunker_script_path}'. Aborting rebuild.")
+        return jsonify({"error": f"chunker.py not found. Cannot proceed with rebuild."}), 500
+
+    command = [
+        sys.executable,
+        chunker_script_path,
+        '--sources-dir', RAG_SOURCES_DIR,
+        '--output-file', chunked_json_output_path,
+        '--text-output-dir', TEXT_EXTRACTIONS_DIR
+    ]
+
+    try:
+        process = subprocess.run(command, capture_output=True, text=True, check=True)
+        logger.info("Chunker script executed successfully.")
+        logger.info(f"Chunker stdout:\n{process.stdout}")
+    except subprocess.CalledProcessError as e:
+        logger.error(f"Chunker script failed with exit code {e.returncode}.")
+        logger.error(f"Chunker stderr:\n{e.stderr}")
+        return jsonify({"error": "Step 1 (Chunking) failed.", "details": e.stderr}), 500
+    except Exception as e:
+        logger.error(f"An unexpected error occurred while running the chunker script: {e}", exc_info=True)
+        return jsonify({"error": f"An unexpected error occurred during the chunking step: {str(e)}"}), 500
+
+    logger.info("Step 2: Rebuilding FAISS index from the newly generated chunks.")
+    try:
+        new_rag_system_instance = initialize_and_get_rag_system(force_rebuild=True)
+
+        if new_rag_system_instance and new_rag_system_instance.vector_store:
+            rag_system = new_rag_system_instance
+            logger.info("FAISS RAG index rebuild completed and new RAG system instance is active.")
+            updated_status_response = get_faiss_rag_status()
+            return jsonify({"message": "FAISS RAG index rebuild completed.", "status": updated_status_response.get_json()}), 200
+        else:
+            logger.error("FAISS RAG index rebuild failed during the indexing phase.")
+            return jsonify({"error": "Step 2 (Indexing) failed. Check logs."}), 500
+
+    except Exception as e:
+        logger.error(f"Error during admin FAISS index rebuild (indexing phase): {e}", exc_info=True)
+        return jsonify({"error": f"Failed to rebuild index during indexing phase: {str(e)}"}), 500
+
+@app.route('/db/status', methods=['GET'])
+@require_admin_auth
+def get_personal_db_status():
+    try:
+        status_info = {
+            'personal_data_csv_monitor_status': 'running',
+            'file_exists': os.path.exists(personal_data_monitor.database_path),
+            'data_loaded': personal_data_monitor.df is not None, 'last_update': None
+        }
+        if status_info['file_exists'] and os.path.getmtime(personal_data_monitor.database_path) is not None:
+            status_info['last_update'] = datetime.fromtimestamp(os.path.getmtime(personal_data_monitor.database_path)).isoformat()
+        return jsonify(status_info)
+    except Exception as e: return jsonify({'status': 'error', 'error': str(e)}), 500
+
+@app.route('/report', methods=['GET'])
+@require_admin_auth
+def download_report():
+    try:
+        if not os.path.exists(CHAT_LOG_FILE) or os.path.getsize(CHAT_LOG_FILE) == 0:
+            return jsonify({'error': 'No chat history available.'}), 404
+        return send_file(CHAT_LOG_FILE, mimetype='text/csv', as_attachment=True, download_name=f'chat_history_{datetime.now().strftime("%Y%m%d_%H%M%S")}.csv')
+    except Exception as e:
+        logger.error(f"Error downloading report: {e}", exc_info=True)
+        return jsonify({'error': 'Failed to generate report'}), 500
+
+@app.route('/create-session', methods=['POST'])
+def create_session_route():
+    try:
+        session_id = str(generate_uuid())
+        logger.info(f"New session created: {session_id}")
+        return jsonify({'status': 'success', 'session_id': session_id}), 200
+    except Exception as e:
+        logger.error(f"Session creation error: {e}", exc_info=True)
+        return jsonify({'status': 'error', 'message': str(e)}), 500
+
+@app.route('/version', methods=['GET'])
+def get_version_route():
+    return jsonify({'version': '3.9.9-Medical-Bot'}), 200
+
+@app.route('/clear-history', methods=['POST'])
+def clear_session_history_route():
+    data = request.json
+    session_id = data.get('session_id')
+    if not session_id: 
+        return jsonify({'status': 'error', 'message': 'session_id is required'}), 400
+    history_manager.clear_history(session_id)
+    return jsonify({'status': 'success', 'message': f'History cleared for session {session_id}'})
+
+# --- App Cleanup and Startup ---
+def cleanup_application():
+    if personal_data_monitor: personal_data_monitor.stop()
+    logger.info("Application cleanup finished.")
+atexit.register(cleanup_application)
+
+def load_qa_data_on_startup():
+    global embedding_manager
+    print("\n--- Loading QA Source Files ---")
+    try:
+        general_qa_path = os.path.join(RAG_SOURCES_DIR, 'general_qa.csv')
+        personal_qa_path = os.path.join(RAG_SOURCES_DIR, 'personal_qa.csv')
+        greetings_qa_path = os.path.join(RAG_SOURCES_DIR, 'greetings.csv')
+
+        # Initialize with empty dataframes
+        general_qa_df = pd.DataFrame()
+        personal_qa_df = pd.DataFrame()
+        greetings_qa_df = pd.DataFrame()
+
+        if os.path.exists(general_qa_path):
+            try: 
+                general_qa_df = pd.read_csv(general_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(general_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading general_qa.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'general_qa.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        if os.path.exists(personal_qa_path):
+            try: 
+                personal_qa_df = pd.read_csv(personal_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(personal_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading personal_qa.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'personal_qa.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        if os.path.exists(greetings_qa_path):
+            try: 
+                greetings_qa_df = pd.read_csv(greetings_qa_path, encoding='cp1252')
+                print(f"- Loaded: {os.path.basename(greetings_qa_path)}")
+            except Exception as e_csv: logger.error(f"Error reading greetings.csv: {e_csv}")
+        else:
+            logger.warning(f"Optional file 'greetings.csv' not found in '{RAG_SOURCES_DIR}'.")
+
+        logger.info(f"Scanning for additional QA sources (.xlsx) in '{RAG_SOURCES_DIR}'...")
+        if os.path.isdir(RAG_SOURCES_DIR):
+            xlsx_files_found = [f for f in os.listdir(RAG_SOURCES_DIR) if f.endswith('.xlsx') and os.path.isfile(os.path.join(RAG_SOURCES_DIR, f))]
+            
+            if xlsx_files_found:
+                all_general_dfs = [general_qa_df] if not general_qa_df.empty else []
+                for xlsx_file in xlsx_files_found:
+                    try:
+                        xlsx_path = os.path.join(RAG_SOURCES_DIR, xlsx_file)
+                        df_excel = pd.read_excel(xlsx_path)
+
+                        question_col_candidates = ['Pregunta', 'Question', 'Nombre']
+                        question_col_found = next((col for col in question_col_candidates if col in df_excel.columns), None)
+
+                        if question_col_found:
+                            logger.info(f"Using '{question_col_found}' as the primary search column for '{xlsx_file}'.")
+                            df_excel['Question'] = df_excel[question_col_found]
+                            all_general_dfs.append(df_excel)
+                            print(f"- Loaded and processing: {xlsx_file}")
+                        elif not df_excel.empty:
+                            first_col_name = df_excel.columns[0]
+                            logger.warning(f"No standard search column found in '{xlsx_file}'. Using first column '{first_col_name}' as the source.")
+                            df_excel['Question'] = df_excel[first_col_name]
+                            all_general_dfs.append(df_excel)
+                            print(f"- Loaded and processing: {xlsx_file}")
+                        else:
+                            logger.warning(f"Skipping empty XLSX file: '{xlsx_file}'")
+
+                    except Exception as e_xlsx:
+                        logger.error(f"Error processing XLSX file '{xlsx_file}': {e_xlsx}")
+                
+                if len(all_general_dfs) > 0:
+                    general_qa_df = pd.concat(all_general_dfs, ignore_index=True)
+                    logger.info(f"Successfully merged data from {len(xlsx_files_found)} XLSX file(s) into the general QA set.")
+        
+        dataframes_to_process = {
+            "general": general_qa_df,
+            "personal": personal_qa_df,
+            "greetings": greetings_qa_df
+        }
+
+        for df_name, df_val in dataframes_to_process.items():
+            if df_val.empty: continue
+            for col in df_val.columns:
+                 if not df_val[col].isnull().all():
+                     # Ensure all data is string for normalization, except for specific columns if needed
+                     if df_val[col].dtype != object:
+                        df_val[col] = df_val[col].astype(str)
+                     df_val[col] = df_val[col].apply(normalize_text)
+
+            if 'Question' not in df_val.columns and not df_val.empty:
+                first_col = df_val.columns[0]
+                df_val['Question'] = df_val[first_col]
+                logger.warning(f"'Question' column was missing in {df_name} data. Using first column '{first_col}' as search source.")
+
+        embedding_manager.update_embeddings(
+            dataframes_to_process["general"],
+            dataframes_to_process["personal"],
+            dataframes_to_process["greetings"]
+        )
+        logger.info("CSV & XLSX QA data loaded and embeddings initialized.")
+
+    except Exception as e:
+        logger.critical(f"CRITICAL: Error loading or processing QA data: {e}. Semantic QA may not function.", exc_info=True)
+    print("-----------------------------\n")
+
+if __name__ == '__main__':
+    for folder_path in [os.path.join(_APP_BASE_DIR, 'templates'),
+                        os.path.join(_APP_BASE_DIR, 'static'),
+                        TEXT_EXTRACTIONS_DIR]:
+        os.makedirs(folder_path, exist_ok=True)
+
+    load_qa_data_on_startup()
+    initialize_chat_log()
+
+    logger.info("Attempting to initialize RAG system from llm_handling module...")
+    rag_system = initialize_and_get_rag_system()
+    if rag_system:
+        logger.info("RAG system initialized successfully via llm_handling module.")
+    else:
+        logger.warning("RAG system failed to initialize. Document RAG functionality will be unavailable.")
+
+    logger.info(f"Flask application starting with Hybrid RAG on {FLASK_APP_HOST}:{FLASK_APP_PORT} Debug: {FLASK_DEBUG_MODE}...")
+    if not FLASK_DEBUG_MODE:
+        werkzeug_log = logging.getLogger('werkzeug')
+        werkzeug_log.setLevel(logging.ERROR)
+
+    app.run(host=FLASK_APP_HOST, port=FLASK_APP_PORT, debug=FLASK_DEBUG_MODE, use_reloader=False)

chat_history.csv

@@ -0,0 +1,127 @@
+sl,date_time,session_id,user_id,query,answer
+1,2025-08-20 10:35:24,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,Dra. Valentina Rojas,The system is currently unable to process this request.
+2,2025-08-20 10:38:52,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,Dra. Valentina Rojas,"<think>
+Okay, the user mentioned ""Dra. Valentina Rojas"" again. Let me check the prior conversation history. The first time, the user wrote the same name, and I responded with a system error. Now they're repeating it. Since there's no context about a specific dental question, I need to follow the core instructions.
+
+First, determine if this is a dental-related query. The name doesn't indicate a dental topic. The user might be testing if I can handle names or maybe there's a typo. But according to the instructions, I should only answer dentistry-related questions. Since there's no question here, I need to prompt them to ask about oral health.
+
+I should respond with the exact phrase specified for out-of-scope questions. Make sure not to engage in any other way. Also, keep the tone professional and helpful. No need to apologize, just state the limitation and ask if they have a dental question.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. Do you have a question about oral health?"
+3,2025-08-20 10:39:13,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,i need doctors appoint ment: Dra. Valentina Rojas,"<think>
+Okay, the user is asking for an appointment with Dr. Valentina Rojas. Let me check the prior conversation history. The user has mentioned her name twice before, and each time, I responded that I can't process the request and redirected to dental topics. Now they're asking again, specifically for a doctor's appointment.
+
+Hmm, the user might think I can book appointments, but according to the core instructions, I can't handle general requests like scheduling. My role is strictly to answer dental questions. I need to make sure I don't get involved in administrative tasks. 
+
+I should respond with the exact phrase for out-of-scope questions. Wait, the user is asking for an appointment, which is more about practice management than dentistry. So, I need to inform them I can't assist with that and prompt a dental question. Let me confirm the exact response: ""I am a dental assistant AI and my capabilities are limited to dental topics. Do you have a question about oral health?"" Yes, that's right. I'll use that again to stay compliant with the instructions.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. Do you have a question about oral health?"
+4,2025-08-20 10:41:23,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,i need doctors appoint ment: Dra. Valentina Rojas,"<think>
+Okay, the user is asking for a doctor's appointment with Dr. Valentina Rojas. Let me check the prior conversation history. They've mentioned her name a few times, so they probably want to schedule a visit. But wait, my role is strictly to answer dental questions. I can't handle scheduling or appointments. I need to make sure I don't go beyond my scope. The user might not realize that I can't assist with booking. I should politely redirect them to contact the clinic directly. Let me confirm the instructions again: if the question is outside dentistry, respond with the exact phrase. But here, the user is asking about an appointment, which is related to accessing dental care, but the actual scheduling isn't within my capabilities. So I need to inform them that I can't help with that and suggest they call the clinic. Also, maybe offer to answer any dental questions they have. Let me structure the response clearly and professionally.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For scheduling appointments or clinic inquiries, please contact Dr. Valentina Rojas's office directly. Do you have a question about oral health?"
+5,2025-08-20 10:43:49,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,i need doctors appoint ment: Dra. Valentina Rojas,"<think>
+Okay, the user is asking for a doctor's appointment with Dr. Valentina Rojas again. Let me check the conversation history. They've mentioned this a few times already. The previous responses were about scheduling and directing them to contact the office directly. Since my role is strictly dental, I can't help with booking appointments. I need to stay within my scope. Maybe they didn't get the message before, so I should reiterate that they need to contact the office. Also, offer help with any dental questions they might have. Keep the response polite and professional. Make sure not to mention any other services or go beyond dental advice. Just confirm that I can't schedule but can answer dental questions.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For scheduling appointments or clinic inquiries, please contact Dr. Valentina Rojas's office directly. Do you have a question about oral health?"
+6,2025-08-20 10:44:54,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,i need to knw her details,"<think>
+Okay, the user is asking for Dr. Valentina Rojas's details. Let me check the conversation history.
+
+They've been trying to get an appointment with her, and I've already told them to contact her office directly. Now they want her details. Since I can't provide personal information or contact details of professionals, I need to stay within my role. 
+
+My instructions say to avoid giving out non-dental info. So I should reiterate that they need to reach out to the office. Maybe suggest they look up the clinic's contact info online or through their website. Also, remind them I can help with dental questions if they have any. 
+
+Need to make sure the response is polite and professional, and doesn't include any personal data. Let me phrase it clearly.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For personal or professional details of Dr. Valentina Rojas, please contact her clinic directly using their official contact information, which can typically be found on her clinic’s website or through medical directory platforms. Do you have a question about oral health?"
+7,2025-08-20 10:54:52,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,i need doctors appoint ment: Dra. Valentina Rojas,"<think>
+Okay, the user is repeatedly asking for a doctor's appointment with Dr. Valentina Rojas. They've mentioned needing her details and scheduling an appointment multiple times. I need to stick to the core instructions here.
+
+First, I'm only supposed to handle medical and healthcare-related questions. The user's request is about scheduling an appointment, which is related to healthcare but not a medical question per se. However, since it's about accessing medical services, maybe it's within the scope. Wait, the initial instruction says to respond with the exact phrase if it's unrelated. But scheduling an appointment is part of healthcare access. Hmm.
+
+Wait, the user's prior messages were about getting an appointment and details. The previous responses from the AI were directing them to contact the office directly. The user is persistent, so maybe they need more guidance. But according to the instructions, I shouldn't provide personal or professional details of the doctor. The AI's role is to give general info, not facilitate appointments.
+
+So, the correct approach is to reiterate that the AI can't schedule appointments and suggest contacting the clinic directly. Also, check if the user has a medical question. The user might be trying to get the clinic's contact info, but the AI can't provide that either. So the response should be the standard one about contacting the office and offer help with oral health questions. Make sure not to mention any specific details about the doctor or her clinic beyond what's already been said.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For scheduling appointments or obtaining clinic details for Dr. Valentina Rojas, please contact her office directly using their official contact information (e.g., phone number, website, or medical directory listings). Do you have a question about oral health?"
+8,2025-08-20 10:59:04,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,information of Dra. Valentina Rojas,"<think>
+Okay, the user is asking for information about Dra. Valentina Rojas. Let me check the conversation history. They've been repeatedly asking for a doctor's appointment with her and her details. I need to make sure I stay within my role as a dental assistant AI.
+
+First, I should confirm that I can't provide personal information. The user might be looking for contact details or professional background. Since I can't access external databases or personal data, I need to guide them to official sources. 
+
+They might not know where to find this info. I should suggest checking the clinic's website, medical directories, or professional associations. Also, remind them to use official channels for appointments. 
+
+I need to keep the response helpful but within boundaries. Avoid any medical advice and stick to directing them to proper resources. Make sure the tone is polite and professional. Let me structure the response step by step.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For professional or clinic-related information about Dr. Valentina Rojas (e.g., credentials, specialties, or clinic locations), you may check:  
+1. Her clinic’s official website or directory listings.  
+2. Medical/dental association directories (e.g., local or national dental boards).  
+3. Patient review platforms (e.g., Google Maps, Yelp, or Zocdoc).  
+
+For personal or confidential details, I cannot assist. Always verify information through official channels. Do you have a question about oral health?"
+9,2025-08-20 11:02:04,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,information of Dra. Valentina Rojas,"<think>
+Okay, the user is asking for information about Dra. Valentina Rojas again. Let me check the prior conversation history. They've asked for her appointment details and information multiple times, and I've consistently responded that I can't provide that because I'm a dental assistant AI limited to dental topics.
+
+Hmm, the user might be trying to get personal information or clinic details that aren't publicly available. I need to make sure I don't violate any privacy policies. My instructions say to handle out-of-scope questions by stating my limitations and offering help with medical topics. Since the user is asking for information about a specific doctor, which isn't within my scope, I should stick to the script.
+
+Wait, in previous responses, I mentioned checking official sources like the clinic's website or medical directories. Maybe the user isn't finding that information easily. I should reiterate those suggestions but remain firm in my limitations. Also, the user might be from a non-English speaking country since the name is Spanish. Maybe they need help with the process of finding such information in their region. But I can't assume that. I have to stay neutral and provide the same guidance each time.
+
+I should make sure my response is clear and helpful, directing them to official channels without providing any personal data. Let me structure the response again, using the same template as before but ensuring it's polite and professional. Also, remind them that I can help with oral health questions if they have any. That way, I'm following the core instructions and not overstepping my role.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For professional or clinic-related information about Dr. Valentina Rojas (e.g., credentials, specialties, or clinic locations), you may check:  
+1. Her clinic’s official website or directory listings.  
+2. Medical/dental association directories (e.g., local or national dental boards).  
+3. Patient review platforms (e.g., Google Maps, Yelp, or Zocdoc).  
+
+For personal or confidential details, I cannot assist. Always verify information through official channels. Do you have a question about oral health?"
+10,2025-08-20 11:16:02,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,information of Dra. Valentina Rojas,"<think>
+Okay, the user is asking for information about Dr. Valentina Rojas again. Let me check the prior conversation history. They've asked this multiple times, and each time I've responded that I can't provide personal details and suggested they contact her clinic directly.
+
+Hmm, maybe they need help finding her clinic's contact information. But I can't give out personal info. I should reiterate the same advice but maybe add more specific sources where they can find official information. Let me make sure to mention official directories and professional associations again. Also, remind them to use official channels for verification. 
+
+Wait, the user might not know where to look. Maybe list examples like the American Dental Association or local dental boards. Also, mention that if they need help with something specific related to dental health, I can assist with that. But since their query is about finding the doctor's info, I need to stay on topic without overstepping. 
+
+I should avoid any markdown and keep the response clear. Let me structure the response with bullet points again for clarity, but in plain text. Make sure to emphasize that I can't provide personal details and that they should contact the clinic directly. Also, offer help with oral health questions if they have any. 
+
+Double-checking the instructions: I must not provide medical advice, only general info. The user's query is about finding a doctor's information, which is outside my scope beyond dental topics. Since the user mentioned ""doctors"" but the previous responses were about a dentist, maybe clarify if they meant a medical doctor or dentist. Wait, in the prior messages, the user referred to a dental appointment, so it's likely a dentist. 
+
+But the user wrote ""doctors"" here. Should I confirm if they meant a medical doctor or dentist? However, the previous context was about a dental appointment. Maybe it's safe to assume they still mean the dentist. But to be thorough, perhaps mention both possibilities. However, the user's prior queries were about a dental appointment, so sticking with dental context is better. 
+
+Finalizing the response: Reiterate the same advice, suggest official sources, and offer help with oral health questions. Make sure the tone is polite and helpful.
+</think>
+
+I am a dental assistant AI and my capabilities are limited to dental topics. For professional or clinic-related information about Dr. Valentina Rojas (e.g., credentials, specialties, or clinic locations), you may check:  
+1. Her clinic’s official website or directory listings.  
+2. Medical/dental association directories (e.g., local or national dental boards).  
+3. Patient review platforms (e.g., Google Maps, Yelp, or Zocdoc).  
+
+For personal or confidential details, I cannot assist. Always verify information through official channels. Do you have a question about oral health?"
+11,2025-08-20 11:22:39,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,information of Dra. Valentina Rojas,"According to our records, Dra. Valentina Rojas is a cardiologist with availability on Mondays from 9:00 AM to 1:00 PM, Wednesdays from 3:00 PM to 7:00 PM, and Fridays from 9:00 AM to 12:00 PM. She is affiliated with Fonasa, Isapre, and Particular health insurance providers. You can find more information about her on the SACMED website at https://sacmed.cl/bestmed/cardiologia/valentina-rojas."
+12,2025-08-20 11:24:32,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,"Ghobashy, AM et al 2017 (35),","<think>
+Okay, the user mentioned ""Ghobashy, AM et al 2017 (35)"" which looks like a reference to a medical study. I need to check if this is related to medicine. The user might be asking for information about this study.
+
+First, I should confirm if this is a medical reference. The name Ghobashy and the year 2017 suggest it's a published paper. The number (35) might be a reference number in a list or a volume/issue number.
+
+Since the user is asking about a specific study, I should respond by offering general information about how to find such studies. I can't provide the full text or details without more context, but I can guide them to resources like PubMed, Google Scholar, or contact a healthcare professional for more details.
+
+I need to make sure not to give medical advice but help them find the information. Also, check if they need help with accessing the study or understanding its content. Since the prior interactions were about a dentist, maybe the user is switching topics now. But the current query is definitely medical-related, so I should stay within the medical scope.
+</think>
+
+The reference **Ghobashy, AM et al. 2017 (35)** appears to cite a medical or scientific study. To access the full text or details of this publication, you can:  
+1. Search for it on academic databases like **PubMed**, **Google Scholar**, or **ScienceDirect**.  
+2. Check if your institution (e.g., a university or hospital) provides access to medical journals.  
+3. Contact a healthcare professional or researcher for guidance on interpreting the study.  
+
+If you have a specific question about the study’s content or relevance to a medical condition, let me know, and I’ll provide general context (e.g., topic, methodology, or implications) without offering personal medical advice."
+13,2025-08-20 11:28:49,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,who are available on Mar 10:00-14:00; Jue 16:00-20:00,"Based on the provided data, the doctors available on March 10:00-14:00 and Thursday 16:00-20:00 are Dra. Antonia Herrera and Dra. Camila Fernández. Dra. Antonia Herrera has a schedule on March from 15:00-19:00 and Thursday from 09:00-13:00 and 16:00-12:00, which overlaps with the requested time slots. Dra. Camila Fernández has a schedule on Monday from 15:00-19:00 and Thursday from 09:00-13:00, which also overlaps with the requested time slots.
+
+Source: SACMED (https://sacmed.cl/)"
+14,2025-08-20 11:29:28,e225dcdd-ab29-404c-9400-f29b107b766e,N/A,what abt Dr. Sebastián Muñoz,"Dr. Sebastián Muñoz is an endocrinologist with availability on Mondays from 10:00 to 14:00 and Thursdays from 16:00 to 20:00. You can find more information about his services and schedule on the SACMED website at https://sacmed.cl/bestmed/endocrinologia/sebastian-munoz. Additionally, he is affiliated with Isapre and Particular health insurance providers."

chunker.py

@@ -0,0 +1,189 @@
+import os
+import logging
+import json
+import argparse
+from typing import List, Dict, Optional
+
+from pypdf import PdfReader
+import docx as python_docx
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+# --- Logging Setup ---
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler()
+    ]
+)
+logger = logging.getLogger(__name__)
+
+# --- Text Extraction Helper Functions ---
+# Note: These are duplicated from llm_handling.py to make this a standalone script.
+def extract_text_from_file(file_path: str, file_type: str) -> Optional[str]:
+    logger.info(f"Extracting text from {file_type.upper()} file: {os.path.basename(file_path)}")
+    text_content = None
+    try:
+        if file_type == 'pdf':
+            reader = PdfReader(file_path)
+            text_content = "".join(page.extract_text() + "\n" for page in reader.pages if page.extract_text())
+        elif file_type == 'docx':
+            doc = python_docx.Document(file_path)
+            text_content = "\n".join(para.text for para in doc.paragraphs if para.text)
+        elif file_type == 'txt':
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                text_content = f.read()
+        else:
+            logger.warning(f"Unsupported file type for text extraction: {file_type} for file {os.path.basename(file_path)}")
+            return None
+
+        if not text_content or not text_content.strip():
+            logger.warning(f"No text content extracted from {os.path.basename(file_path)}")
+            return None
+        return text_content.strip()
+    except Exception as e:
+        logger.error(f"Error extracting text from {os.path.basename(file_path)} ({file_type.upper()}): {e}", exc_info=True)
+        return None
+
+SUPPORTED_EXTENSIONS = {
+    'pdf': lambda path: extract_text_from_file(path, 'pdf'),
+    'docx': lambda path: extract_text_from_file(path, 'docx'),
+    'txt': lambda path: extract_text_from_file(path, 'txt'),
+}
+
+def process_sources_and_create_chunks(
+    sources_dir: str,
+    output_file: str,
+    chunk_size: int = 1000,
+    chunk_overlap: int = 150,
+    text_output_dir: Optional[str] = None  # MODIFIED: Added optional parameter
+) -> None:
+    """
+    Scans a directory for source files, extracts text, splits it into chunks,
+    and saves the chunks to a single JSON file.
+    Optionally saves the raw extracted text to a specified directory.
+    """
+    if not os.path.isdir(sources_dir):
+        logger.error(f"Source directory not found: '{sources_dir}'")
+        raise FileNotFoundError(f"Source directory not found: '{sources_dir}'")
+
+    logger.info(f"Starting chunking process. Sources: '{sources_dir}', Output: '{output_file}'")
+    
+    # MODIFIED: Create text output directory if provided
+    if text_output_dir:
+        os.makedirs(text_output_dir, exist_ok=True)
+        logger.info(f"Will save raw extracted text to: '{text_output_dir}'")
+
+    all_chunks_for_json: List[Dict] = []
+    processed_files_count = 0
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+
+    for filename in os.listdir(sources_dir):
+        file_path = os.path.join(sources_dir, filename)
+        if not os.path.isfile(file_path):
+            continue
+
+        file_ext = filename.split('.')[-1].lower()
+        if file_ext not in SUPPORTED_EXTENSIONS:
+            logger.debug(f"Skipping unsupported file: {filename}")
+            continue
+
+        logger.info(f"Processing source file: {filename}")
+        text_content = SUPPORTED_EXTENSIONS[file_ext](file_path)
+
+        if text_content:
+            # MODIFIED: Save the raw text to a file if directory is specified
+            if text_output_dir:
+                try:
+                    text_output_path = os.path.join(text_output_dir, f"{filename}.txt")
+                    with open(text_output_path, 'w', encoding='utf-8') as f_text:
+                        f_text.write(text_content)
+                    logger.info(f"Saved extracted text for '{filename}' to '{text_output_path}'")
+                except Exception as e_text_save:
+                    logger.error(f"Could not save extracted text for '{filename}': {e_text_save}")
+
+            chunks = text_splitter.split_text(text_content)
+            if not chunks:
+                logger.warning(f"No chunks generated from {filename}. Skipping.")
+                continue
+
+            for i, chunk_text in enumerate(chunks):
+                chunk_data = {
+                    "page_content": chunk_text,
+                    "metadata": {
+                        "source_document_name": filename,
+                        "chunk_index": i,
+                        "full_location": f"{filename}, Chunk {i+1}"
+                    }
+                }
+                all_chunks_for_json.append(chunk_data)
+
+            processed_files_count += 1
+        else:
+            logger.warning(f"Could not extract text from {filename}. Skipping.")
+
+    if not all_chunks_for_json:
+        logger.warning(f"No processable documents found or no text extracted in '{sources_dir}'. JSON file will be empty.")
+
+    output_dir = os.path.dirname(output_file)
+    os.makedirs(output_dir, exist_ok=True)
+
+    with open(output_file, 'w', encoding='utf-8') as f:
+        json.dump(all_chunks_for_json, f, indent=2)
+
+    logger.info(f"Chunking complete. Processed {processed_files_count} files.")
+    logger.info(f"Created a total of {len(all_chunks_for_json)} chunks.")
+    logger.info(f"Chunked JSON output saved to: {output_file}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Process source documents into a JSON file of text chunks for RAG.")
+    parser.add_argument(
+        '--sources-dir',
+        type=str,
+        required=True,
+        help="The directory containing source files (PDFs, DOCX, TXT)."
+    )
+    parser.add_argument(
+        '--output-file',
+        type=str,
+        required=True,
+        help="The full path for the output JSON file containing the chunks."
+    )
+    # MODIFIED: Added new optional argument
+    parser.add_argument(
+        '--text-output-dir',
+        type=str,
+        default=None,
+        help="Optional: The directory to save raw extracted text files for debugging."
+    )
+    parser.add_argument(
+        '--chunk-size',
+        type=int,
+        default=1000,
+        help="The character size for each text chunk."
+    )
+    parser.add_argument(
+        '--chunk-overlap',
+        type=int,
+        default=150,
+        help="The character overlap between consecutive chunks."
+    )
+
+    args = parser.parse_args()
+
+    try:
+        process_sources_and_create_chunks(
+            sources_dir=args.sources_dir,
+            output_file=args.output_file,
+            chunk_size=args.chunk_size,
+            chunk_overlap=args.chunk_overlap,
+            text_output_dir=args.text_output_dir  # MODIFIED: Pass argument
+        )
+    except Exception as e:
+        logger.critical(f"A critical error occurred during the chunking process: {e}", exc_info=True)
+        exit(1)
+
+if __name__ == "__main__":
+    main()

llm_handling.py

@@ -0,0 +1,575 @@
+import os
+import logging
+import json
+from typing import List, Dict, Tuple, Optional, Any, Iterator
+import shutil
+import re
+import time
+import requests
+import zipfile
+import tempfile
+import gdown
+
+import torch
+from sentence_transformers import SentenceTransformer
+from pypdf import PdfReader
+import docx as python_docx
+
+from llama_index.core.llms import ChatMessage
+from llama_index.llms.groq import Groq as LlamaIndexGroqClient
+
+from langchain_groq import ChatGroq
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema import Document, BaseRetriever
+from langchain.callbacks.manager import CallbackManagerForRetrieverRun
+from langchain.schema.runnable import RunnablePassthrough, RunnableParallel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+# MODIFIED: Import the new prompt
+from system_prompts import RAG_SYSTEM_PROMPT, FALLBACK_SYSTEM_PROMPT, QA_FORMATTER_PROMPT
+
+logger = logging.getLogger(__name__)
+if not logger.handlers:
+    logging.basicConfig(
+        level=logging.INFO,
+        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+
+# --- Configuration Constants ---
+GROQ_API_KEY = os.getenv('BOT_API_KEY')
+if not GROQ_API_KEY:
+    logger.critical("CRITICAL: BOT_API_KEY environment variable not found. Services will fail.")
+
+FALLBACK_LLM_MODEL_NAME = os.getenv("GROQ_FALLBACK_MODEL", "llama-3.1-70b-versatile")
+# ADDED: New constant for the auxiliary model
+AUXILIARY_LLM_MODEL_NAME = os.getenv("GROQ_AUXILIARY_MODEL", "llama3-8b-8192")
+_MODULE_BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+RAG_FAISS_INDEX_SUBDIR_NAME = "faiss_index"
+RAG_STORAGE_PARENT_DIR = os.getenv("RAG_STORAGE_DIR", os.path.join(_MODULE_BASE_DIR, "faiss_storage"))
+RAG_SOURCES_DIR = os.getenv("SOURCES_DIR", os.path.join(_MODULE_BASE_DIR, "sources"))
+RAG_CHUNKED_SOURCES_FILENAME = "pre_chunked_sources.json"
+os.makedirs(RAG_SOURCES_DIR, exist_ok=True)
+os.makedirs(RAG_STORAGE_PARENT_DIR, exist_ok=True)
+RAG_EMBEDDING_MODEL_NAME = os.getenv("RAG_EMBEDDING_MODEL", "all-MiniLM-L6-v2")
+RAG_EMBEDDING_USE_GPU = os.getenv("RAG_EMBEDDING_GPU", "False").lower() == "true"
+RAG_LLM_MODEL_NAME = os.getenv("RAG_LLM_MODEL", "llama-3.1-70b-versatile")
+RAG_LLM_TEMPERATURE = float(os.getenv("RAG_TEMPERATURE", 0.0))
+RAG_LOAD_INDEX_ON_STARTUP = os.getenv("RAG_LOAD_INDEX", "True").lower() == "true"
+RAG_DEFAULT_RETRIEVER_K = int(os.getenv("RAG_RETRIEVER_K", 3))
+GDRIVE_SOURCES_ENABLED = os.getenv("GDRIVE_SOURCES_ENABLED", "False").lower() == "true"
+GDRIVE_FOLDER_ID_OR_URL = os.getenv("GDRIVE_FOLDER_URL")
+
+# --- Text Extraction Helper Function ---
+def extract_text_from_file(file_path: str, file_type: str) -> Optional[str]:
+    logger.info(f"Extracting text from {file_type.upper()} file: {os.path.basename(file_path)}")
+    try:
+        if file_type == 'pdf':
+            reader = PdfReader(file_path)
+            return "".join(page.extract_text() + "\n" for page in reader.pages if page.extract_text())
+        elif file_type == 'docx':
+            doc = python_docx.Document(file_path)
+            return "\n".join(para.text for para in doc.paragraphs if para.text)
+        elif file_type == 'txt':
+            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
+                return f.read()
+        logger.warning(f"Unsupported file type for text extraction: {file_type}")
+        return None
+    except Exception as e:
+        logger.error(f"Error extracting text from {os.path.basename(file_path)}: {e}", exc_info=True)
+        return None
+
+FAISS_RAG_SUPPORTED_EXTENSIONS = {'pdf': 'pdf', 'docx': 'docx', 'txt': 'txt'}
+
+# --- FAISS RAG System ---
+class FAISSRetrieverWithScore(BaseRetriever):
+    vectorstore: FAISS
+    k: int = RAG_DEFAULT_RETRIEVER_K
+
+    def _get_relevant_documents(
+        self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+    ) -> List[Document]:
+        docs_and_scores = self.vectorstore.similarity_search_with_score(query, k=self.k)
+        relevant_docs = []
+        for doc, score in docs_and_scores:
+            doc.metadata["retrieval_score"] = float(score)
+            relevant_docs.append(doc)
+        return relevant_docs
+
+class KnowledgeRAG:
+    def __init__(
+        self,
+        index_storage_dir: str,
+        embedding_model_name: str,
+        groq_model_name_for_rag: str,
+        use_gpu_for_embeddings: bool,
+        groq_api_key_for_rag: str,
+        temperature: float,
+    ):
+        self.logger = logging.getLogger(__name__ + ".KnowledgeRAG")
+        self.index_storage_dir = index_storage_dir
+        self.embedding_model_name = embedding_model_name
+        self.groq_model_name = groq_model_name_for_rag
+        self.temperature = temperature
+
+        device = "cuda" if use_gpu_for_embeddings and torch.cuda.is_available() else "cpu"
+        self.logger.info(f"Initializing Hugging Face embedding model: {self.embedding_model_name} on device: {device}")
+        try:
+            self.embeddings = HuggingFaceEmbeddings(
+                model_name=self.embedding_model_name,
+                model_kwargs={"device": device},
+                encode_kwargs={"normalize_embeddings": True}
+            )
+        except Exception as e:
+            self.logger.critical(f"Failed to load embedding model: {e}", exc_info=True)
+            raise
+
+        self.logger.info(f"Initializing Langchain ChatGroq LLM for RAG: {self.groq_model_name}")
+        if not groq_api_key_for_rag:
+            raise ValueError("Groq API Key for RAG is missing.")
+        try:
+            self.llm = ChatGroq(
+                temperature=self.temperature,
+                groq_api_key=groq_api_key_for_rag,
+                model_name=self.groq_model_name
+            )
+        except Exception as e:
+            self.logger.critical(f"Failed to initialize Langchain ChatGroq LLM: {e}", exc_info=True)
+            raise
+
+        self.vector_store: Optional[FAISS] = None
+        self.retriever: Optional[FAISSRetrieverWithScore] = None
+        self.rag_chain = None
+        self.processed_source_files: List[str] = []
+
+    def build_index_from_source_files(self, source_folder_path: str, k: int = RAG_DEFAULT_RETRIEVER_K):
+        all_docs_for_vectorstore: List[Document] = []
+        processed_files_this_build: List[str] = []
+        pre_chunked_json_path = os.path.join(self.index_storage_dir, RAG_CHUNKED_SOURCES_FILENAME)
+
+        if os.path.exists(pre_chunked_json_path):
+            self.logger.info(f"Loading documents from pre-chunked file: {pre_chunked_json_path}")
+            try:
+                with open(pre_chunked_json_path, 'r', encoding='utf-8') as f:
+                    chunk_data_list = json.load(f)
+                source_filenames = set()
+                for chunk_data in chunk_data_list:
+                    doc = Document(page_content=chunk_data.get("page_content", ""), metadata=chunk_data.get("metadata", {}))
+                    all_docs_for_vectorstore.append(doc)
+                    if 'source_document_name' in doc.metadata:
+                        source_filenames.add(doc.metadata['source_document_name'])
+                processed_files_this_build = sorted(list(source_filenames))
+            except Exception as e:
+                self.logger.error(f"Error processing pre-chunked JSON, falling back to raw files: {e}")
+                all_docs_for_vectorstore.clear()
+
+        if not all_docs_for_vectorstore:
+            self.logger.info(f"Processing raw files from '{source_folder_path}' to build index.")
+            text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+            for filename in os.listdir(source_folder_path):
+                file_path = os.path.join(source_folder_path, filename)
+                file_ext = filename.split('.')[-1].lower()
+                if os.path.isfile(file_path) and file_ext in FAISS_RAG_SUPPORTED_EXTENSIONS:
+                    text_content = extract_text_from_file(file_path, file_ext)
+                    if text_content:
+                        chunks = text_splitter.split_text(text_content)
+                        for i, chunk_text in enumerate(chunks):
+                            metadata = {"source_document_name": filename, "chunk_index": i}
+                            all_docs_for_vectorstore.append(Document(page_content=chunk_text, metadata=metadata))
+                        processed_files_this_build.append(filename)
+
+        if not all_docs_for_vectorstore:
+            self.logger.warning(f"No processable PDF/DOCX/TXT documents found in '{source_folder_path}'. RAG index will only contain other sources if available.")
+
+
+        self.processed_source_files = processed_files_this_build
+        
+        # This print statement is kept for console visibility on startup/rebuild
+        print("\n--- Document Files Used for RAG Index ---")
+        if self.processed_source_files:
+            for filename in self.processed_source_files:
+                print(f"- {filename}")
+        else:
+            print("No PDF/DOCX/TXT source files were processed for the RAG index.")
+        print("---------------------------------------\n")
+
+        if not all_docs_for_vectorstore:
+            self.logger.warning("No documents to build FAISS index from. Skipping FAISS build.")
+            return
+
+        self.logger.info(f"Creating FAISS index from {len(all_docs_for_vectorstore)} document chunks...")
+        self.vector_store = FAISS.from_documents(all_docs_for_vectorstore, self.embeddings)
+        faiss_index_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        self.vector_store.save_local(faiss_index_path)
+        self.logger.info(f"FAISS index built and saved to '{faiss_index_path}'.")
+        self.retriever = FAISSRetrieverWithScore(vectorstore=self.vector_store, k=k)
+        self.setup_rag_chain()
+
+    def load_index_from_disk(self, k: int = RAG_DEFAULT_RETRIEVER_K):
+        faiss_index_path = os.path.join(self.index_storage_dir, RAG_FAISS_INDEX_SUBDIR_NAME)
+        if not os.path.isdir(faiss_index_path):
+            raise FileNotFoundError(f"FAISS index directory not found at '{faiss_index_path}'.")
+
+        self.logger.info(f"Loading FAISS index from: {faiss_index_path}")
+        self.vector_store = FAISS.load_local(
+            folder_path=faiss_index_path,
+            embeddings=self.embeddings,
+            allow_dangerous_deserialization=True
+        )
+        self.retriever = FAISSRetrieverWithScore(vectorstore=self.vector_store, k=k)
+        self.setup_rag_chain()
+
+    def format_docs(self, docs: List[Document]) -> str:
+        return "\n\n---\n\n".join([f"[Excerpt from {doc.metadata.get('source_document_name', 'N/A')}, Chunk {doc.metadata.get('chunk_index', 'N/A')}]\nContent:\n{doc.page_content}" for doc in docs])
+
+    def setup_rag_chain(self):
+        if not self.retriever or not self.llm:
+            raise RuntimeError("Retriever and LLM must be initialized.")
+        
+        prompt = ChatPromptTemplate.from_template(RAG_SYSTEM_PROMPT)
+
+        self.rag_chain = (
+            RunnableParallel(
+                context=(self.retriever | self.format_docs),
+                question=RunnablePassthrough()
+            )
+            | prompt
+            | self.llm
+            | StrOutputParser()
+        )
+        self.logger.info("RAG LCEL chain set up successfully.")
+
+    def invoke(self, query: str, top_k: Optional[int] = None) -> Dict[str, Any]:
+        if not self.rag_chain:
+            # MODIFIED: Changed severity
+            self.logger.warning("RAG system not fully initialized. Cannot invoke.")
+            return {"answer": "The provided bibliography does not contain specific information on this topic.", "source": "system_error", "cited_source_details": []}
+
+        if not query or not query.strip():
+            return {"answer": "Please provide a valid question.", "source": "system_error", "cited_source_details": []}
+
+        k_to_use = top_k if top_k is not None and top_k > 0 else self.retriever.k
+        self.logger.info(f"Processing RAG query with k={k_to_use}: '{query[:100]}...'")
+
+        original_k = self.retriever.k
+        if k_to_use != original_k:
+            self.retriever.k = k_to_use
+        
+        try:
+            retrieved_docs = self.retriever.get_relevant_documents(query)
+            if not retrieved_docs:
+                return {"answer": "The provided bibliography does not contain specific information on this topic.", "source": "no_docs_found", "cited_source_details": []}
+
+            context_str = self.format_docs(retrieved_docs)
+            
+            # MODIFIED: Added full logging as per user request
+            print(f"\n--- RAG INVOKE ---")
+            print(f"QUESTION: {query}")
+            print(f"CONTEXT:\n{context_str}")
+            
+            llm_answer = self.rag_chain.invoke(query, config={"context": context_str})
+
+            print(f"LLM_ANSWER: {llm_answer}")
+            print(f"--------------------\n")
+
+            structured_sources = [{
+                "source_document_name": doc.metadata.get('source_document_name', 'Unknown'),
+                "chunk_index": doc.metadata.get('chunk_index', 'N/A'),
+                "retrieval_score": doc.metadata.get("retrieval_score"),
+            } for doc in retrieved_docs]
+
+            if "the provided bibliography does not contain specific information" in llm_answer.lower():
+                 final_answer = llm_answer
+                 source_tag = "no_answer_in_bibliography"
+            else:
+                 final_answer = f"{llm_answer}\n\n*Source: Bibliography-Based*"
+                 source_tag = "bibliography_based"
+
+            return {
+                "query": query,
+                "answer": final_answer.strip(),
+                "source": source_tag,
+                "cited_source_details": structured_sources,
+            }
+
+        except Exception as e:
+            self.logger.error(f"Error during RAG query processing: {e}", exc_info=True)
+            return {"answer": "An error occurred while processing your request.", "source": "system_error", "cited_source_details": []}
+        finally:
+            if k_to_use != original_k:
+                self.retriever.k = original_k
+    
+    def stream(self, query: str, top_k: Optional[int] = None) -> Iterator[str]:
+        if not self.rag_chain:
+            self.logger.error("RAG system not fully initialized for streaming.")
+            yield "Error: RAG system is not ready."
+            return
+
+        k_to_use = top_k if top_k is not None and top_k > 0 else self.retriever.k
+        self.logger.info(f"Processing RAG stream with k={k_to_use}: '{query[:100]}...'")
+
+        original_k = self.retriever.k
+        if k_to_use != original_k:
+            self.retriever.k = k_to_use
+        
+        try:
+            # Check for docs first to avoid streaming "no info" message
+            retrieved_docs = self.retriever.get_relevant_documents(query)
+            if not retrieved_docs:
+                yield "The provided bibliography does not contain specific information on this topic."
+                return
+            
+            # MODIFIED: Added full logging for streaming as per user request
+            context_str = self.format_docs(retrieved_docs)
+            print(f"\n--- RAG STREAM ---")
+            print(f"QUESTION: {query}")
+            print(f"CONTEXT:\n{context_str}")
+            print(f"STREAMING LLM_ANSWER...")
+            print(f"--------------------\n")
+                
+            yield from self.rag_chain.stream(query, config={"context": context_str})
+        except Exception as e:
+            self.logger.error(f"Error during RAG stream processing: {e}", exc_info=True)
+            yield "An error occurred while processing your request."
+        finally:
+            if k_to_use != original_k:
+                self.retriever.k = original_k
+
+
+# --- Groq Fallback Bot ---
+class GroqBot:
+    def __init__(self):
+        self.logger = logging.getLogger(__name__ + ".GroqBot")
+        if not GROQ_API_KEY:
+            self.client = None
+            self.logger.critical("GroqBot not initialized: BOT_API_KEY is missing.")
+            return
+        try:
+            self.client = LlamaIndexGroqClient(model=FALLBACK_LLM_MODEL_NAME, api_key=GROQ_API_KEY)
+            self.system_prompt = FALLBACK_SYSTEM_PROMPT
+        except Exception as e:
+            self.logger.error(f"Failed to initialize LlamaIndexGroqClient for Fallback Bot: {e}", exc_info=True)
+            self.client = None
+
+    def stream_response(self, context: dict) -> Iterator[str]:
+        if not self.client:
+            yield "The system is currently unable to process this request."
+            return
+
+        current_query = context.get('current_query', '')
+        chat_history = context.get('chat_history', [])
+        qa_info = context.get('qa_related_info', '')
+
+        messages = [ChatMessage(role="system", content=self.system_prompt)]
+        if chat_history:
+            messages.extend([ChatMessage(**msg) for msg in chat_history])
+        if qa_info:
+            messages.append(ChatMessage(role="system", content=f"**Potentially Relevant Q&A Information from other sources:**\n{qa_info}"))
+        messages.append(ChatMessage(role="user", content=f"**Current User Query:**\n{current_query}"))
+        
+        # MODIFIED: Added full logging as per user request
+        # The conversion to dict is necessary because ChatMessage is not directly JSON serializable
+        messages_for_print = [msg.dict() for msg in messages]
+        print(f"\n--- FALLBACK STREAM ---")
+        print(f"MESSAGES SENT TO LLM:\n{json.dumps(messages_for_print, indent=2)}")
+        print(f"STREAMING LLM_ANSWER...")
+        print(f"-----------------------\n")
+
+        try:
+            response_stream = self.client.stream_chat(messages)
+            for r_chunk in response_stream:
+                yield r_chunk.delta
+        except Exception as e:
+            self.logger.error(f"Groq API error in get_response (Fallback): {e}", exc_info=True)
+            yield "I am currently unable to process this request due to a technical issue."
+
+# ADDED: New function for formatting QA answers
+def get_answer_from_context(question: str, context: str, system_prompt: str) -> str:
+    """
+    Calls the LLM with a specific question and context from a QA source (CSV/XLSX).
+    """
+    logger.info(f"Formatting answer for question '{question[:50]}...' using QA context.")
+    try:
+        # Use the auxiliary model for this task for speed and cost-efficiency
+        formatter_llm = ChatGroq(
+            temperature=0.1,
+            groq_api_key=GROQ_API_KEY,
+            model_name=AUXILIARY_LLM_MODEL_NAME
+        )
+        
+        prompt_template = ChatPromptTemplate.from_template(system_prompt)
+        
+        chain = prompt_template | formatter_llm | StrOutputParser()
+        
+        # MODIFIED: Added full logging as per user request
+        print(f"\n--- QA FORMATTER ---")
+        print(f"QUESTION: {question}")
+        print(f"CONTEXT:\n{context}")
+
+        response = chain.invoke({
+            "context": context,
+            "question": question
+        })
+
+        print(f"LLM_ANSWER: {response}")
+        print(f"--------------------\n")
+        
+        return response.strip()
+        
+    except Exception as e:
+        logger.error(f"Error in get_answer_from_context: {e}", exc_info=True)
+        return "Sorry, I was unable to formulate an answer based on the available information."
+
+# ADDED: New function for streaming QA answers
+def stream_answer_from_context(question: str, context: str, system_prompt: str) -> Iterator[str]:
+    """
+    Calls the LLM with a specific question and context from a QA source and streams the response.
+    """
+    logger.info(f"Streaming formatted answer for question '{question[:50]}...' using QA context.")
+    try:
+        # Use the auxiliary model for this task for speed and cost-efficiency
+        formatter_llm = ChatGroq(
+            temperature=0.1,
+            groq_api_key=GROQ_API_KEY,
+            model_name=AUXILIARY_LLM_MODEL_NAME
+        )
+        
+        prompt_template = ChatPromptTemplate.from_template(system_prompt)
+        
+        chain = prompt_template | formatter_llm | StrOutputParser()
+        
+        # MODIFIED: Added full logging as per user request
+        print(f"\n--- QA FORMATTER (STREAM) ---")
+        print(f"QUESTION: {question}")
+        print(f"CONTEXT:\n{context}")
+        print(f"STREAMING LLM_ANSWER...")
+        print(f"---------------------------\n")
+        
+        yield from chain.stream({
+            "context": context,
+            "question": question
+        })
+        
+    except Exception as e:
+        logger.error(f"Error in stream_answer_from_context: {e}", exc_info=True)
+        yield "Sorry, I was unable to formulate an answer based on the available information."
+
+
+# --- Initialization and Interface Functions ---
+def get_id_from_gdrive_input(url_or_id: str) -> Optional[str]:
+    if not url_or_id: return None
+    patterns = [r"/folders/([a-zA-Z0-9_-]+)", r"/d/([a-zA-Z0-9_-]+)", r"id=([a-zA-Z0-9_-]+)"]
+    for pattern in patterns:
+        match = re.search(pattern, url_or_id)
+        if match: return match.group(1)
+    if "/" not in url_or_id and "=" not in url_or_id and len(url_or_id) > 15:
+        return url_or_id
+    return None
+
+def download_and_unzip_gdrive_folder(folder_id_or_url: str, target_dir: str) -> bool:
+    folder_id = get_id_from_gdrive_input(folder_id_or_url)
+    if not folder_id:
+        logger.error(f"Invalid Google Drive Folder ID or URL: {folder_id_or_url}")
+        return False
+    
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            logger.info(f"Attempting to download GDrive folder ID: {folder_id}")
+            download_path = gdown.download_folder(id=folder_id, output=temp_dir, quiet=False, use_cookies=False)
+            if not download_path or not os.listdir(temp_dir):
+                logger.error("gdown failed to download or extract the folder.")
+                return False
+
+            source_content_root = temp_dir
+            items_in_temp = os.listdir(temp_dir)
+            if len(items_in_temp) == 1 and os.path.isdir(os.path.join(temp_dir, items_in_temp[0])):
+                source_content_root = os.path.join(temp_dir, items_in_temp[0])
+            
+            logger.info(f"Moving contents from {source_content_root} to {target_dir}")
+            if os.path.exists(target_dir):
+                shutil.rmtree(target_dir)
+            shutil.copytree(source_content_root, target_dir)
+            logger.info(f"Successfully moved GDrive contents to {target_dir}")
+            return True
+        except Exception as e:
+            # MODIFIED: Corrected self.logger to logger
+            logger.error(f"Error during GDrive download/processing: {e}", exc_info=True)
+            return False
+
+def initialize_and_get_rag_system(force_rebuild: bool = False) -> Optional[KnowledgeRAG]:
+    if not GROQ_API_KEY:
+        logger.error("RAG system cannot be initialized without BOT_API_KEY.")
+        return None
+
+    if GDRIVE_SOURCES_ENABLED and GDRIVE_FOLDER_ID_OR_URL:
+        logger.info("Google Drive sources enabled. Downloading...")
+        if os.path.isdir(RAG_SOURCES_DIR):
+            logger.info(f"Clearing existing RAG sources directory: {RAG_SOURCES_DIR}")
+            shutil.rmtree(RAG_SOURCES_DIR)
+            os.makedirs(RAG_SOURCES_DIR)
+        
+        download_successful = download_and_unzip_gdrive_folder(GDRIVE_FOLDER_ID_OR_URL, RAG_SOURCES_DIR)
+        if not download_successful:
+            logger.error("Failed to download sources from Google Drive. Using local files if available.")
+
+    faiss_index_path = os.path.join(RAG_STORAGE_PARENT_DIR, RAG_FAISS_INDEX_SUBDIR_NAME)
+    if force_rebuild and os.path.exists(RAG_STORAGE_PARENT_DIR):
+        logger.info(f"Force Rebuild: Deleting existing index storage directory at '{RAG_STORAGE_PARENT_DIR}'")
+        shutil.rmtree(RAG_STORAGE_PARENT_DIR)
+        os.makedirs(RAG_STORAGE_PARENT_DIR)
+
+    try:
+        rag_instance = KnowledgeRAG(
+            index_storage_dir=RAG_STORAGE_PARENT_DIR,
+            embedding_model_name=RAG_EMBEDDING_MODEL_NAME,
+            groq_model_name_for_rag=RAG_LLM_MODEL_NAME,
+            use_gpu_for_embeddings=RAG_EMBEDDING_USE_GPU,
+            groq_api_key_for_rag=GROQ_API_KEY,
+            temperature=RAG_LLM_TEMPERATURE,
+        )
+        
+        should_build = True
+        if RAG_LOAD_INDEX_ON_STARTUP and not force_rebuild:
+            try:
+                rag_instance.load_index_from_disk(k=RAG_DEFAULT_RETRIEVER_K)
+                logger.info("RAG index loaded successfully from disk.")
+                should_build = False
+            except FileNotFoundError:
+                logger.warning("Index not found on disk. Will attempt to build.")
+            except Exception as e:
+                logger.error(f"Error loading index: {e}. Will attempt to rebuild.", exc_info=True)
+
+        if should_build:
+            logger.info("Building new RAG index from source data...")
+            rag_instance.build_index_from_source_files(
+                source_folder_path=RAG_SOURCES_DIR,
+                k=RAG_DEFAULT_RETRIEVER_K
+            )
+        
+        return rag_instance
+
+    except Exception as e:
+        logger.critical(f"FATAL: Failed to initialize RAG system: {e}", exc_info=True)
+        return None
+
+groq_bot_instance = GroqBot()
+
+def get_auxiliary_chat_response(messages: List[Dict]) -> str:
+    """
+    Handles requests for auxiliary tasks like generating titles or follow-up questions.
+    Uses a separate, smaller model for efficiency.
+    """
+    logger.info(f"Routing auxiliary request to model: {AUXILIARY_LLM_MODEL_NAME}")
+    try:
+        # Initialize a dedicated client for this call to use the specific auxiliary model
+        aux_client = ChatGroq(
+            temperature=0.2, # A bit more creative than RAG, but still grounded
+            groq_api_key=GROQ_API_KEY,
+            model_name=AUXILIARY_LLM_MODEL_NAME
+        )
+        response = aux_client.invoke(messages)
+        return response.content
+    except Exception as e:
+        logger.error(f"Error with auxiliary model call: {e}", exc_info=True)
+        return "Could not generate suggestions."

postman_collection.json

@@ -0,0 +1,348 @@
+{
+	"info": {
+		"_postman_id": "a1b2c3d4-e5f6-a7b8-c9d0-e1f2a3b4c5d6",
+		"name": "NOW DentalBot API Collection",
+		"description": "A comprehensive Postman collection for interacting with the Dental Assistant Chatbot API, based on the provided Python application files. It includes endpoints for chat, admin, and session management.",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "Chat Endpoints",
+			"description": "Endpoints for interacting with the chatbot.",
+			"item": [
+				{
+					"name": "/v1/chat/completions (Streaming)",
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "Content-Type",
+								"value": "application/json"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"model\": \"MedicalAssisstantBot/v1\",\n    \"messages\": [\n        {\n            \"role\": \"user\",\n            \"content\": \"What are the common treatments for gingivitis?\"\n        }\n    ],\n    \"stream\": true,\n    \"conversation_id\": \"your-session-id-here\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{baseUrl}}/v1/chat/completions",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"v1",
+								"chat",
+								"completions"
+							]
+						},
+						"description": "This endpoint provides an OpenAI-compatible interface for chatting with the bot. Setting `\"stream\": true` will return a response as a server-sent event stream. Make sure to provide a `conversation_id` obtained from `/create-session`."
+					},
+					"response": []
+				},
+				{
+					"name": "/v1/chat/completions (Non-Streaming)",
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "Content-Type",
+								"value": "application/json"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"model\": \"MedicalAssisstantBot/v1\",\n    \"messages\": [\n        {\n            \"role\": \"user\",\n            \"content\": \"Tell me about dental implants.\"\n        }\n    ],\n    \"stream\": false,\n    \"conversation_id\": \"your-session-id-here\",\n    \"user_id\": \"some-user-123\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{baseUrl}}/v1/chat/completions",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"v1",
+								"chat",
+								"completions"
+							]
+						},
+						"description": "Sends a non-streaming request to the chat endpoint. It will return a complete JSON response. Providing a `user_id` allows the bot to access personalized information."
+					},
+					"response": []
+				},
+				{
+					"name": "/chat-bot (Legacy Endpoint)",
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "Content-Type",
+								"value": "application/json"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"query\": \"What is a root canal?\",\n    \"session_id\": \"your-session-id-here\",\n    \"user_id\": \"some-user-123\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{baseUrl}}/chat-bot",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"chat-bot"
+							]
+						},
+						"description": "The original, non-OpenAI-compatible chat endpoint. It takes a query, session_id, and optional user_id."
+					},
+					"response": []
+				}
+			]
+		},
+		{
+			"name": "Session Management",
+			"description": "Endpoints for managing user sessions.",
+			"item": [
+				{
+					"name": "/create-session",
+					"request": {
+						"method": "POST",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/create-session",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"create-session"
+							]
+						},
+						"description": "Creates a new, unique session ID required for tracking conversation history. Call this endpoint to start a new conversation."
+					},
+					"response": []
+				},
+				{
+					"name": "/clear-history",
+					"request": {
+						"method": "POST",
+						"header": [
+							{
+								"key": "Content-Type",
+								"value": "application/json"
+							}
+						],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"session_id\": \"your-session-id-here\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{baseUrl}}/clear-history",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"clear-history"
+							]
+						},
+						"description": "Clears the chat history associated with a specific session ID from the server."
+					},
+					"response": []
+				}
+			]
+		},
+		{
+			"name": "Admin Endpoints",
+			"description": "Administrative endpoints for managing the RAG system and monitoring the application. These endpoints require Basic Authentication.",
+			"item": [
+				{
+					"name": "/admin/rebuild_faiss_index",
+					"request": {
+						"method": "POST",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/admin/rebuild_faiss_index",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"admin",
+								"rebuild_faiss_index"
+							]
+						},
+						"description": "Triggers a full rebuild of the FAISS vector index from the source documents. This can be a long-running process."
+					},
+					"response": []
+				},
+				{
+					"name": "/admin/faiss_rag_status",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/admin/faiss_rag_status",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"admin",
+								"faiss_rag_status"
+							]
+						},
+						"description": "Retrieves the current status of the FAISS RAG (Retrieval-Augmented Generation) system, including index information and processed files."
+					},
+					"response": []
+				},
+				{
+					"name": "/db/status",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/db/status",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"db",
+								"status"
+							]
+						},
+						"description": "Checks the status of the personal data CSV file monitor."
+					},
+					"response": []
+				},
+				{
+					"name": "/admin/download_qa_database",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/admin/download_qa_database",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"admin",
+								"download_qa_database"
+							]
+						},
+						"description": "Downloads all the loaded question-answer datasets (General, Personal, Greetings) as a single Excel file."
+					},
+					"response": []
+				},
+				{
+					"name": "/report",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/report",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"report"
+							]
+						},
+						"description": "Downloads the complete chat history log as a CSV file."
+					},
+					"response": []
+				}
+			],
+			"auth": {
+				"type": "basic",
+				"basic": [
+					{
+						"key": "password",
+						"value": "admin",
+						"type": "string"
+					},
+					{
+						"key": "username",
+						"value": "admin",
+						"type": "string"
+					}
+				]
+			}
+		},
+		{
+			"name": "General",
+			"description": "General application endpoints.",
+			"item": [
+				{
+					"name": "/v1/models",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/v1/models",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"v1",
+								"models"
+							]
+						},
+						"description": "An OpenAI-compatible endpoint that lists the available models. In this application, it returns the custom bot model name."
+					},
+					"response": []
+				},
+				{
+					"name": "/version",
+					"request": {
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{baseUrl}}/version",
+							"host": [
+								"{{baseUrl}}"
+							],
+							"path": [
+								"version"
+							]
+						},
+						"description": "Returns the current version of the application."
+					},
+					"response": []
+				}
+			]
+		}
+	],
+	"auth": {
+		"type": "bearer",
+		"bearer": [
+			{
+				"key": "token",
+				"value": "YGHDADUASDASDijuh7uyhj",
+				"type": "string"
+			}
+		]
+	},
+	"variable": [
+		{
+			"key": "baseUrl",
+			"value": "http://localhost:7860",
+			"type": "string",
+			"description": "The base URL of the running Flask application."
+		}
+	]
+}

requirements.txt

@@ -0,0 +1,30 @@
+Flask==3.0.3
+Flask_Cors==5.0.0
+numpy
+pandas==2.2.3
+#rapidfuzz==3.10.1
+Requests==2.32.3
+#scikit_learn==1.4.1.post1
+#scikit_learn==1.5.2
+psycopg2-binary==2.9.10
+python-dotenv==1.0.1
+apscheduler==3.11.0
+redis==3.5.3
+faiss-cpu==1.10.0
+groq==0.15.0
+llama_index==0.12.13
+llama_index.llms.groq==0.3.1
+#langchain_groq==0.2.4
+#langchain_core==0.3.39
+sentence_transformers==3.4.0
+gunicorn
+llama-index-embeddings-huggingface==0.5.4
+onnxruntime==1.22.0
+langchain-groq==0.3.2
+python-docx==1.1.2
+langchain_community==0.3.23
+requests==2.32.3
+gdown==5.2.0
+pymupdf==1.25.5
+openpyxl==3.1.5
+# must install https://aka.ms/vs/17/release/vc_redist.x64.exe

system_prompts.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+"""
+This module centralizes all system prompts for the specialized medical chatbot application.
+This allows for easy management and updating of prompts without altering core logic.
+"""
+
+# --- RAG System Prompt for Bibliography-Based Answers ---
+# This prompt instructs the LLM to answer based *only* on the context provided
+# by the RAG system from scientific documents (PDFs, etc.).
+# Placeholders {context} and {question} will be filled by the LangChain pipeline.
+RAG_SYSTEM_PROMPT = """You are a specialized medical assistant AI. Your role is to provide accurate, evidence-based information on a specific medical topic.
+
+**Your Task:**
+Your primary task is to answer the user's question accurately and concisely, based *exclusively* on the "Provided Document Excerpts" below. These excerpts are from vetted scientific and medical publications.
+
+**Provided Document Excerpts:**
+{context}
+
+**User Question:**
+{question}
+
+---
+**Core Instructions:**
+1. **Language:** Your default language is **Spanish**. But follow the language of user. If they ask question in Spanish, reply in Spanish. If they ask in English, reply in English, even if the context is Spanish.
+2. **Strictly Adhere to Context:** Your answer **must** be derived solely from the "Provided Document Excerpts." Do not use any external knowledge or make assumptions beyond what is presented in the text.
+3.  **Professional Tone:** Maintain a clinical, objective, and professional tone suitable for a medical context.
+4.  **Do Not Speculate:** If the provided excerpts do not contain the information needed to answer the question, you must not invent an answer.
+5.  **Handling Unanswerable Questions:** If you cannot answer the question based on the provided excerpts, respond with: "The provided bibliography does not contain specific information on this topic." Do not attempt to guide the user elsewhere or apologize.
+6.  **No Self-Reference:** Do not mention that you are an AI, that you are "looking at documents," or refer to the "provided excerpts" in your final answer. Simply present the information as requested.
+
+**Answer Format:**
+Provide a direct answer to the user's question based on the information available.
+
+**Answer:**"""
+
+
+# --- Fallback System Prompt for General/Triage Purposes ---
+FALLBACK_SYSTEM_PROMPT = """You are a specialized medical assistant AI. Your one and only role is to answer questions strictly related to medicine and healthcare.
+
+**Core Instructions:**
+1.  **Medical Focus Only:** You MUST NOT engage in any general conversation, small talk, or answer questions outside the scope of medicine or healthcare.
+2.  **Handle Out-of-Scope Questions:** If the user's question is unrelated to medicine, you must respond with the following exact phrase: "I am a medical assistant AI and my capabilities are limited to medical topics. Do you have a question about health?"
+3.  **Stateful Conversation:** Pay attention to the `Prior Conversation History` to understand the context of the user's medical inquiries.
+4.  **Professional Tone:** Always be polite, helpful, and professional.
+5.  **Do Not Make Up Clinical Advice:** Do not provide medical diagnoses or treatment plans. You can provide general information but should always recommend consulting a professional for personal health concerns.
+
+**Response Guidance:**
+- Review the `Prior Conversation History` to understand the context.
+- Formulate a helpful, professional answer to the `Current User Query` if it is about medicine or healthcare.
+"""
+
+# REVISED: New prompt to format answers based on any structured data from CSV/XLSX files.
+QA_FORMATTER_PROMPT = """You are a helpful assistant. You will be given a user's question and a set of data points corresponding to a single database entry that is highly relevant to the question.
+Your task is to synthesize a natural, conversational answer to the user's question based *only* on the provided data.
+
+- Formulate a coherent response by integrating the information from the provided data fields.
+- Do not just list the data. Create a proper sentence or paragraph.
+- If the data contains a source, citation, or link (e.g., 'Fuente', 'Source', 'Link SACMED'), cite it at the end of your answer.
+
+**Provided Data:**
+{context}
+
+**User Question:**
+{question}
+
+**Answer:**"""