Reorganized the project: -Documents hosted on Firestore db, -conversations saved

.gitattributes

@@ -1,2 +1 @@
 documents/*.pdf filter=lfs diff=lfs merge=lfs -text
-chroma_db/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -6,4 +6,10 @@ all-libraries.txt
 # Ignore ChromaDB persistent storage
 chroma_db/
 
-temp*
+temp*
+
+__pycache__/
+
+atemp/
+
+documents/

README.md

@@ -24,7 +24,7 @@ It uses:
 ## How to Use:
 This Space exposes a `/compassia/` API endpoint. You can interact with it using `curl`, Postman, Insomnia, or by integrating it with your Next.js frontend.
 
-### API Endpoint: `/ask-pdf/` (POST request)
+### API Endpoint: `/compassia/` (POST request)
 **Request Body (JSON):**
 ```json
 {

app.py

@@ -1,67 +1,90 @@
-# app.py
-
 import os
 from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 import uvicorn
 import sys
+import json
+import base64
+
+# Add the 'src' directory to the Python path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'src')))
+
+# Import components from the new modular structure, specifically from src.compassia
+from src.config import CHROMADB_PERSIST_DIRECTORY, CHROMADB_COLLECTION_NAME # Removed PDF_DOCUMENT_PATHS as it's handled in rag_system
+from src.compassia import DocumentRAG, embedding_model, initialize_firebase_client, db_firestore # Import db_firestore and related functions from compassia.py
 
-# Add src to the Python path so we can import modules from it
-# This is crucial for deployment environments where 'src' might not be automatically recognized
-#sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'src')))
+# --- Firebase Initialization (Global, once per process) ---
+# Initialize Firebase Admin SDK using a secret from Hugging Face Spaces
+# This function is now called directly from app.py startup.
+# db_firestore is already imported and will be set by initialize_firebase_client()
+initialize_firebase_client()
 
-# Import your DocumentRAG class and other necessary components from your backend script
-# Make sure your rag_backend.py has `embedding_model` defined globally or passed correctly
-from compassia import DocumentRAG, embedding_model, pdf_document_paths, extract_text_from_pdf, ocr_pdf, chunk_text # Import all needed functions/variables
 
 # --- Initialize the RAG system globally ---
-# This ensures the model loads and indexing happens once when the FastAPI app starts
-# and persists across requests within the same process.
-# ChromaDB will save its data to the './chroma_db' directory within the Space.
 print("--- FastAPI App Startup: Initializing RAG System ---")
 rag_system = DocumentRAG(
     embedding_model=embedding_model,
-    persist_directory="./chroma_db", # ChromaDB will store data here in the Space
-    collection_name="pdf_documents_collection",
-    chunk_size=700, # Match your existing chunk size
-    overlap=100     # Match your existing overlap
+    persist_directory=CHROMADB_PERSIST_DIRECTORY,
+    collection_name=CHROMADB_COLLECTION_NAME
 )
 
 # --- Index documents on startup ---
 # This loop will run when the FastAPI app first starts.
 # It uses ChromaDB's persistence, so documents already indexed will be skipped.
-print("--- FastAPI App Startup: Indexing Documents (ChromaDB persistence) ---")
-for pdf_path in pdf_document_paths:
-    # Ensure the path is correct relative to the Space's filesystem
-    full_pdf_path = os.path.join(os.path.dirname(__file__), pdf_path)
-    if os.path.exists(full_pdf_path):
-        rag_system.add_document(full_pdf_path) # Call add_document on rag_system
-    else:
-        print(f"API Error: PDF file not found at {full_pdf_path}. Ensure it's deployed with your app in the 'documents' folder.")
+# Now fetches document URLs directly from Firestore using db_firestore
+print("--- FastAPI App Startup: Indexing Documents from Firestore ---")
+if db_firestore:
+    try:
+        docs_ref = db_firestore.collection('documents').stream()
+        firestore_pdf_infos = []
+        for doc in docs_ref:
+            doc_data = doc.to_dict()
+            if 'fileUrl' in doc_data and doc_data['fileUrl'].endswith('.pdf'):
+                pdf_url = doc_data['fileUrl']
+                display_name = doc_data.get('name_en', os.path.basename(pdf_url))
+                firestore_pdf_infos.append({"url": pdf_url, "name": display_name})
+                print(f"Found PDF in Firestore: {display_name} ({pdf_url})")
+
+        if firestore_pdf_infos:
+            for pdf_info in firestore_pdf_infos:
+                rag_system.add_document(pdf_info['url'], pdf_info['name'])
+        else:
+            print("No PDF documents found in Firestore collection 'documents'.")
+    except Exception as e:
+        print(f"API Error: Error fetching documents from Firestore: {e}")
+        print("Please ensure your Firestore database is accessible and the service account key (FIREBASE_CONFIG_BASE64 secret) is correct.")
+        # If document fetching fails, consider if the app should still start or crash.
+        # For now, it will print the error but continue to try to start the API.
+else:
+    print("API Error: Firestore not initialized. Cannot fetch documents from Firestore on startup.")
+    print("Ensure FIREBASE_CONFIG_BASE64 secret is correctly set.")
+
+
 print("--- FastAPI App Startup: Document indexing complete ---")
 
 
 # --- FastAPI Application Instance ---
 app = FastAPI(
     title="CompassIA",
-    description="Backend API for querying PDFs using DeepSeek (via OpenRouter) and BGE-M3 embeddings.",
+    description="Backend API for querying PDFs using DeepSeek (via OpenRouter) and BGE-M3 embeddings, with conversational memory.",
     version="0.1.0",
 )
 
 # Pydantic model for request body validation
 class QueryRequest(BaseModel):
     question: str
+    conversation_id: str = None # Optional for new conversations
 
 # --- API Endpoint Definition ---
 @app.post("/compassia/")
 async def compassia_endpoint(request: QueryRequest):
     """
-    Answers a question about the indexed PDF documents using RAG.
+    Answers a question about the indexed PDF documents using RAG, with conversational memory.
     """
     try:
-        # Pass an empty list for pdf_paths as documents are already indexed in ChromaDB
-        answer = rag_system.answer_question(request.question, [])
-        return {"answer": answer}
+        # Pass conversation_id to the answer_question function
+        answer = rag_system.answer_question(request.question, conversation_id=request.conversation_id)
+        return {"answer": answer, "conversation_id": request.conversation_id}
     except Exception as e:
         print(f"Error processing /compassia/ request: {e}")
         raise HTTPException(status_code=500, detail=str(e))
@@ -70,9 +93,3 @@ async def compassia_endpoint(request: QueryRequest):
 @app.get("/")
 async def root():
     return {"message": "CompassIA API is running. Use /compassia/ for queries."}
-
-# You can run this locally for testing:
-# if __name__ == "__main__":
-#     # This part runs locally if you execute app.py directly
-#     # For deployment, uvicorn is typically run via a command line.
-#     uvicorn.run(app, host="0.0.0.0", port=8000)

compassia.py

@@ -1,385 +0,0 @@
-import sys
-# IMPORTANT: These lines MUST be at the very top of compassia.py
-# They ensure that any subsequent import of 'sqlite3' (even indirectly by chromadb)
-# will use the version provided by pysqlite3-binary.
-try:
-    import pysqlite3
-    sys.modules['sqlite3'] = pysqlite3
-except ImportError:
-    pass # Fallback if pysqlite3 isn't available, but it should be in Docker
-
-import requests
-import os
-import io
-import re
-import uuid # For generating unique IDs for ChromaDB
-from PIL import Image
-
-# For text extraction from PDFs (non-OCR)
-from pdfminer.high_level import extract_text_to_fp
-from pdfminer.layout import LAParams
-
-# For image-based PDFs (OCR)
-from pdf2image import convert_from_path
-import pytesseract
-
-# For embeddings and vector search
-from FlagEmbedding import BGEM3FlagModel
-import chromadb # pip install chromadb
-
-# --- IMPORTANT: Configure Paths for Tesseract and Poppler ---
-# If Tesseract is not in your system's PATH, uncomment and set this locally:
-# pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
-
-# If pdf2image gives errors about poppler, uncomment and set this locally:
-# poppler_path = r'C:\path\to\poppler\bin'
-
-# --- OpenRouter DeepSeek API Configuration ---
-API_KEY = os.getenv("DEEPSEEK_R1_V3_API_KEY")
-if API_KEY:
-    API_KEY = API_KEY.strip()
-
-if not API_KEY:
-    raise ValueError("API key is not set. Please set the DEEPSEEK_R1_V3_API_KEY environment variable with your OpenRouter key.")
-
-API_URL = 'https://openrouter.ai/api/v1/chat/completions'
-HEADERS = {
-    'Authorization': f'Bearer {API_KEY}',
-    'Content-Type': 'application/json'
-}
-
-# --- Embedding Model Configuration (Local BGE-M3) ---
-print("Loading FlagEmbedding (BGE-M3) model...")
-try:
-    embedding_model = BGEM3FlagModel('BAAI/bge-m3', use_fp16=True)
-    print("FlagEmbedding (BGE-M3) model loaded successfully.")
-except Exception as e:
-    print(f"Error loading FlagEmbedding model: {e}")
-    print("Ensure you have resolved disk space issues for model download and have enough memory.")
-    print("You might need to adjust 'use_fp16' based on your hardware (e.g., False for CPU/older GPUs).")
-    exit(1)
-
-# --- PDF Processing Functions ---
-
-def extract_text_from_pdf(pdf_path: str) -> str:
-    """
-    Extracts text from a PDF. Tries direct text extraction first.
-    If sparse text is found (suggesting image-based PDF), it performs OCR.
-    """
-    print(f"Attempting direct text extraction from: {pdf_path}")
-    output_string = io.StringIO()
-    with open(pdf_path, 'rb') as fp:
-        try:
-            extract_text_to_fp(fp, output_string, laparams=LAParams())
-            text = output_string.getvalue()
-            if len(text.strip()) < 100 and os.path.getsize(pdf_path) > 10000:
-                print("Direct extraction yielded sparse text. Attempting OCR...")
-                return ocr_pdf(pdf_path)
-            return text
-        except Exception as e:
-            print(f"Direct PDF text extraction failed ({e}). Attempting OCR...")
-            return ocr_pdf(pdf_path)
-
-def ocr_pdf(pdf_path: str) -> str:
-    """
-    Performs OCR on a PDF file using pdf2image and pytesseract.
-    Requires Tesseract and Poppler to be installed and in system PATH.
-    """
-    all_text = []
-    try:
-        images = convert_from_path(pdf_path, dpi=300)
-
-        print(f"  Performing OCR on {len(images)} pages...")
-        for i, img in enumerate(images):
-            # Tesseract language packs:
-            # 'eng' for English, 'tur' for Turkish
-            # If you have scanned PDFs in Arabic or French, you MUST install
-            # 'tesseract-ocr-ara' and 'tesseract-ocr-fra' in your Dockerfile
-            # and change 'lang' to 'eng+tur+ara+fra'.
-            page_text = pytesseract.image_to_string(img, lang='eng+tur')
-            all_text.append(page_text)
-            print(f"    Page {i+1} OCR complete.")
-
-    except Exception as e:
-        print(f"OCR process failed: {e}")
-        print("Please ensure Tesseract OCR and Poppler are correctly installed and their executables are in your system's PATH.")
-        return ""
-
-    return "\n".join(all_text)
-
-def chunk_text(text: str, max_chunk_size: int = 700, overlap: int = 100) -> list[str]:
-    """
-    Splits text into chunks of a maximum size with optional overlap.
-    Aims to split by paragraphs/sentences first, then by word.
-    """
-    if not text:
-        return []
-
-    paragraphs = re.split(r'\n\s*\n', text)
-    chunks = []
-    current_chunk = []
-    current_chunk_len = 0
-
-    for para in paragraphs:
-        if not para.strip():
-            continue
-
-        if current_chunk_len + len(para) + len('\n\n') > max_chunk_size:
-            if current_chunk:
-                chunks.append("\n\n".join(current_chunk))
-                current_chunk = []
-                current_chunk_len = 0
-
-            if len(para) > max_chunk_size:
-                words = para.split(' ')
-                sub_chunk = []
-                sub_chunk_len = 0
-                for word in words:
-                    if sub_chunk_len + len(word) + len(' ') > max_chunk_size:
-                        chunks.append(" ".join(sub_chunk))
-                        sub_chunk = [word]
-                        sub_chunk_len = len(word)
-                    else:
-                        sub_chunk.append(word)
-                        sub_chunk_len += len(word) + len(' ')
-                if sub_chunk:
-                    chunks.append(" ".join(sub_chunk))
-            else:
-                current_chunk.append(para)
-                current_chunk_len += len(para) + len('\n\n')
-        else:
-            current_chunk.append(para)
-            current_chunk_len += len(para) + len('\n\n')
-
-    if current_chunk:
-        chunks.append("\n\n".join(current_chunk))
-
-    final_chunks_with_overlap = []
-    for i in range(len(chunks)):
-        chunk = chunks[i]
-        if i > 0 and overlap > 0:
-            prev_chunk_part = chunks[i-1][-overlap:]
-            chunk = prev_chunk_part + "\n" + chunk
-        final_chunks_with_overlap.append(chunk)
-
-    return final_chunks_with_overlap
-
-# --- RAG Core Functions with ChromaDB ---
-
-class DocumentRAG:
-    def __init__(self, embedding_model, persist_directory="./chroma_db", collection_name="pdf_docs", chunk_size=700, overlap=100):
-        self.embedding_model = embedding_model
-        self.chunk_size = chunk_size
-        self.overlap = overlap
-        self.persist_directory = persist_directory
-        self.collection_name = collection_name
-        
-        print(f"Initializing ChromaDB at: {self.persist_directory}")
-        self.client = chromadb.PersistentClient(path=self.persist_directory)
-        
-        self.collection = self.client.get_or_create_collection(
-            name=self.collection_name,
-            metadata={"hnsw:space": "cosine"}
-        )
-        print(f"ChromaDB collection '{self.collection_name}' ready.")
-
-    def _generate_chunk_id(self, pdf_path: str, chunk_idx: int) -> str:
-        return f"{os.path.basename(pdf_path)}_{chunk_idx}_{uuid.uuid4().hex}"
-
-    def add_document(self, pdf_path: str):
-        print(f"Adding document: {pdf_path}")
-        
-        results = self.collection.get(
-            where={"source": pdf_path},
-            limit=1
-        )
-        if results and results['ids']:
-            print(f"  Document '{pdf_path}' already in ChromaDB. Skipping re-indexing.")
-            return
-
-        extracted_text = extract_text_from_pdf(pdf_path)
-        if not extracted_text:
-            print(f"Warning: No text extracted from {pdf_path}. Skipping.")
-            return
-
-        chunks = chunk_text(extracted_text, self.chunk_size, self.overlap)
-        if not chunks:
-            print(f"Warning: No chunks generated for {pdf_path}. Skipping.")
-            return
-
-        documents_to_add = []
-        metadatas_to_add = []
-        ids_to_add = []
-
-        print(f"  Generating embeddings for {len(chunks)} chunks and preparing for ChromaDB...")
-        
-        encoded_results = self.embedding_model.encode(
-            chunks, 
-            batch_size=32,
-            return_dense=True, 
-            return_sparse=False, 
-            return_colbert_vecs=False
-        )
-        
-        chunk_embeddings = encoded_results["dense_vecs"]
-
-        for i, chunk in enumerate(chunks):
-            unique_id = self._generate_chunk_id(pdf_path, i)
-            documents_to_add.append(chunk)
-            metadatas_to_add.append({"source": pdf_path, "chunk_id": i})
-            ids_to_add.append(unique_id)
-        
-        self.collection.add(
-            documents=documents_to_add,
-            embeddings=chunk_embeddings.tolist(),
-            metadatas=metadatas_to_add,
-            ids=ids_to_add
-        )
-        
-        print(f"  {len(documents_to_add)} chunks from '{pdf_path}' added to ChromaDB.")
-        print(f"  Total chunks in collection: {self.collection.count()}")
-
-    def retrieve_context(self, query: str, top_k: int = 3) -> list[str]:
-        """
-        Retrieves top_k most relevant document chunks for a given query from ChromaDB.
-        """
-        if self.collection.count() == 0:
-            print("Error: No documents indexed in ChromaDB. Cannot retrieve context.")
-            return []
-
-        print(f"Retrieving context for query: '{query}'")
-        
-        query_embedding_result = self.embedding_model.encode(
-            [query], 
-            batch_size=1,
-            return_dense=True, 
-            return_sparse=False, 
-            return_colbert_vecs=False
-        )
-        query_embedding = query_embedding_result["dense_vecs"].tolist()
-
-        results = self.collection.query(
-            query_embeddings=query_embedding,
-            n_results=top_k,
-            include=['documents', 'distances', 'metadatas']
-        )
-        
-        retrieved_chunks_texts = []
-        if results and results['documents']:
-            for i, doc_text in enumerate(results['documents'][0]):
-                source_info = results['metadatas'][0][i].get('source', 'Unknown Source')
-                chunk_id_info = results['metadatas'][0][i].get('chunk_id', 'N/A')
-                distance_info = results['distances'][0][i]
-
-                retrieved_chunks_texts.append(doc_text)
-                print(f"  Retrieved chunk {i+1} (distance: {distance_info:.4f}) from '{source_info}' (chunk {chunk_id_info}).")
-        else:
-            print("  No relevant chunks found in ChromaDB.")
-            
-        return retrieved_chunks_texts
-
-    def answer_question(self, question: str, pdf_paths: list[str]) -> str:
-        """
-        Answers a question by ensuring PDFs are indexed, retrieving context,
-        and querying DeepSeek.
-        """
-        for path in pdf_paths:
-            self.add_document(path)
-
-        context_chunks = self.retrieve_context(question)
-        context = "\n\n".join(context_chunks)
-
-        if not context:
-            print("Warning: No relevant context found. Answering based on general knowledge or indicating lack of information.")
-            context_prompt = ""
-        else:
-            context_prompt = f"Using the following context:\n\n{context}\n\n"
-
-        # --- UPDATED SYSTEM PROMPT FOR COMPASSIA AI ---
-        system_prompt = """
-            You are CompassIA, the intelligent assistant for MaarifCompass, committed to supporting Turkiye Maarif Foundation graduates residing in Turkiye.
-
-            Your core function is to deliver precise, document-backed information concerning their needs, primarily focusing on:
-            - University application procedures, requirements, tuition fees, and scholarship opportunities
-            - Accommodation and housing resources
-            - Career networking and professional development
-            - Relevant administrative and support services.
-            - Information related to Turkiye Maarif Foundation, Turkiye Scholarship and more.
-
-            You operate exclusively with data from a designated Document Center. **It is imperative that every piece of information you provide is directly sourced and verifiable from these internal documents.**
-
-            **Should a query fall outside the scope of the provided documents or lack a direct answer within them, you are required to politely inform the user that the specific information is not available in your current knowledge base, without offering any external insights or assumptions.**
-
-            Your answers should be highly accurate, directly relevant, easy to understand, and always prioritize the user's query based strictly on documented facts.
-            **Remember, you always answer the user with the language of the question.**
-        """
-
-        messages = [
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": f"{context_prompt}Question: {question}"}
-        ]
-
-        print("\nSending request to DeepSeek API...")
-        data = {
-            "model": "deepseek/deepseek-chat:free",
-            "messages": messages,
-            "temperature": 0.5,
-            "max_tokens": 500,
-        }
-
-        response = requests.post(API_URL, json=data, headers=HEADERS)
-
-        if response.status_code == 200:
-            ai_response = response.json()
-            answer = ai_response['choices'][0]['message']['content']
-            print("\nDeepSeek Response:")
-            print(answer)
-            return answer
-        else:
-            error_message = f"Failed to fetch data from DeepSeek API. Status Code: {response.status_code}. Response: {response.text}"
-            print(error_message)
-            return f"Error: Could not get an answer from the AI. Details: {error_message}"
-
-# --- Define PDF documents (MOVED TO GLOBAL SCOPE) ---
-pdf_document_paths = [
-    "documents/heracles_tr.pdf",
-    "documents/heracles_en.pdf",
-    "documents/ogrenci_katki_payi_ogrenim_ucretleri.pdf",
-    "documents/Ogrenci_Liderligi_Burs_Programi_Sozlesme_Metni_2024-2025.pdf",
-    "documents/tmv-bursluluk-yonergesi.pdf"
-]
-
-
-# --- Main execution logic ---
-if __name__ == "__main__":
-    rag_system = DocumentRAG(
-        embedding_model=embedding_model,
-        persist_directory="./chroma_db",
-        collection_name="pdf_documents_collection",
-        chunk_size=700,
-        overlap=100
-    )
-    """
-    pdf_document_paths = [
-        "documents/heracles_tr.pdf",
-        "documents/heracles_en.pdf",
-        "documents/ogrenci_katki_payi_ogrenim_ucretleri.pdf",
-        "documents/Ogrenci_Liderligi_Burs_Programi_Sozlesme_Metni_2024-2025.pdf",
-        "documents/tmv-bursluluk-yonergesi.pdf"
-    ]
-    """
-    print("\n--- Indexing Documents ---")
-    for pdf_path in pdf_document_paths:
-        if os.path.exists(pdf_path):
-            rag_system.add_document(pdf_path)
-        else:
-            print(f"Error: PDF file not found at {pdf_path}. Please check the path.")
-
-    print("\n--- Chat With CompassIA (Type 'quit' to exit) ---")
-    while True:
-        user_question = input("\nHow can I help you? ")
-        if user_question.lower() == 'quit':
-            print("Exiting chat.")
-            break
-        
-        rag_system.answer_question(user_question, pdf_document_paths)

documents/Ogrenci_Liderligi_Burs_Programi_Sozlesme_Metni_2024-2025.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fd1dc09026b3212985f74ff6c9f322f9c40883039911b034c84a879ddb1531bd
-size 177096

documents/heracles_en.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:010a6ea71343b7d41f5b0a907f7487448e72c5dad1d37ab333e0b936d4bf5c4a
-size 581483

documents/heracles_tr.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e9879afb2856c8f8c5177ea1470607f5b4a45b49ffa6fd241b24e657a2f8e7a8
-size 564377

documents/ogrenci_katki_payi_ogrenim_ucretleri.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:eb0d72f118f8cf4c8bf8c3212ced62de48c3808b4325ce473cbd1c4418594e8d
-size 549275

documents/tmv-bursluluk-yonergesi.pdf

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aa5e97a0f06393484000d42b9258ca68ecf823b777661064c44683dc7602963d
-size 337112

requirements.txt

@@ -10,9 +10,11 @@ faiss-cpu
 chromadb
 fastapi
 uvicorn # For serving the FastAPI application
-pysqlite3-binary
+#pysqlite3-binary
 
 # System dependencies for Tesseract and Poppler on Linux
 # Hugging Face Spaces uses apt-get for these
 #apt_packages = python3-dev libtesseract-dev libleptonica-dev poppler-utils
 
+firebase-admin
+firebase

src/compassia.py

@@ -0,0 +1,464 @@
+import sys
+# IMPORTANT: These lines MUST be at the very top of compassia.py
+# They ensure that any subsequent import of 'sqlite3' (even indirectly by chromadb)
+# will use the version provided by pysqlite3-binary.
+"""
+try:
+    import pysqlite3
+    sys.modules['sqlite3'] = pysqlite3
+except ImportError:
+    # This should not happen in the Docker environment as pysqlite3-binary is in requirements.txt
+    print("Warning: pysqlite3 not found. Falling back to default sqlite3. ChromaDB might fail if it's too old.")
+    pass
+"""
+import requests
+import os
+import uuid
+import json
+import base64 # For decoding Firebase config
+import hashlib # For hashing URLs for chunk IDs
+import urllib.parse # For parsing URLs
+import io # Import the io module for BytesIO
+
+# Firebase Admin SDK for Firestore
+import firebase_admin
+from firebase_admin import credentials, firestore
+
+# For embeddings and vector search
+from FlagEmbedding import BGEM3FlagModel
+import chromadb
+
+# Import configurations and prompt from local modules
+# These imports assume 'src' directory is on the Python path or script is run from 'src'
+from config import (
+    DEEPSEEK_API_URL, DEEPSEEK_HEADERS,
+    EMBEDDING_MODEL_NAME, EMBEDDING_MODEL_USE_FP16,
+    CHROMADB_PERSIST_DIRECTORY, CHROMADB_COLLECTION_NAME,
+    CHUNK_SIZE, CHUNK_OVERLAP,
+    LLM_TEMPERATURE, LLM_MAX_TOKENS, LLM_HISTORY_MAX_TOKENS,
+    FIREBASE_CONFIG_BASE64
+)
+from pdf_processing import extract_text_from_pdf, chunk_text # Import functions from pdf_processing
+from prompt import SYSTEM_PROMPT # Import the system prompt
+
+
+# --- Global Firebase Firestore Client ---
+FIRESTORE_DATABASE = None
+
+def initialize_firebase_client():
+    """Initializes Firebase Admin SDK and returns the Firestore client."""
+    global FIRESTORE_DATABASE
+    # Check if Firebase Admin SDK is already initialized
+    if not firebase_admin._apps:
+        # Only attempt to initialize if FIREBASE_CONFIG_BASE64 is provided
+        if FIREBASE_CONFIG_BASE64:
+            try:
+                cred_json = base64.b64decode(FIREBASE_CONFIG_BASE64).decode('utf-8')
+                cred_dict = json.loads(cred_json)
+                cred = credentials.Certificate(cred_dict)
+                firebase_admin.initialize_app(cred)
+                print("Firebase Admin SDK initialized successfully.")
+                FIRESTORE_DATABASE = firestore.client()
+                print("Firestore client initialized successfully.")
+                return FIRESTORE_DATABASE
+            except Exception as e:
+                print(f"Error initializing Firebase Admin SDK: {e}")
+                print("Please ensure FIREBASE_CONFIG_BASE64 is correctly set and is a valid Base64-encoded Service Account JSON.")
+                FIRESTORE_DATABASE = None
+                return None
+        else:
+            print("Warning: FIREBASE_CONFIG_BASE64 environment variable not found. Firestore will not be available.")
+            FIRESTORE_DATABASE = None
+            return None
+    else: # Already initialized
+        print("Firebase Admin SDK already initialized.")
+        FIRESTORE_DATABASE = firestore.client() # Ensure global variable is set if already initialized
+        return FIRESTORE_DATABASE
+
+# --- Embedding Model Initialization ---
+print("Loading FlagEmbedding (BGE-M3) model...")
+try:
+    embedding_model = BGEM3FlagModel(EMBEDDING_MODEL_NAME, use_fp16=EMBEDDING_MODEL_USE_FP16)
+    print("FlagEmbedding (BGE-M3) model loaded successfully.")
+except Exception as e:
+    print(f"Error loading FlagEmbedding model: {e}")
+    print("Ensure disk space and memory are sufficient for model download.")
+    print("You might need to adjust 'use_fp16' based on your hardware (e.g., False for CPU/older GPUs).")
+    exit(1)
+
+
+class DocumentRAG:
+    def __init__(self, embedding_model, persist_directory=CHROMADB_PERSIST_DIRECTORY, collection_name=CHROMADB_COLLECTION_NAME):
+        self.embedding_model = embedding_model
+        self.persist_directory = persist_directory
+        self.collection_name = collection_name
+        self.chunk_size = CHUNK_SIZE
+        self.overlap = CHUNK_OVERLAP
+        
+        print(f"Initializing ChromaDB at: {self.persist_directory}")
+        self.client = chromadb.PersistentClient(path=self.persist_directory)
+        
+        self.collection = self.client.get_or_create_collection(
+            name=self.collection_name,
+            metadata={"hnsw:space": "cosine"}
+        )
+        print(f"ChromaDB collection '{self.collection_name}' ready. Total chunks: {self.collection.count()}")
+
+    def _generate_chunk_id(self, pdf_url: str, chunk_idx: int) -> str:
+        """Generates a unique ID for each chunk based on PDF URL and index."""
+        import hashlib
+        url_hash = hashlib.sha256(pdf_url.encode()).hexdigest()[:10]
+        return f"{url_hash}_{chunk_idx}_{uuid.uuid4().hex}"
+
+    def add_document(self, pdf_url: str, document_name: str = None):
+        """
+        Adds a PDF document to the RAG system, processing and indexing its content.
+        Downloads the PDF from the URL.
+        """
+        # Determine display name from parsed URL path if not provided
+        parsed_url_path = urllib.parse.urlparse(pdf_url).path
+        display_name = document_name if document_name else os.path.basename(parsed_url_path)
+        print(f"Adding document from URL: {pdf_url} (Display Name: {display_name})")
+        
+        results = self.collection.get(
+            where={"source": pdf_url},
+            limit=1
+        )
+        if results and results['ids']:
+            print(f"  Document '{display_name}' (from {pdf_url}) already in ChromaDB. Skipping re-indexing.")
+            return
+
+        try:
+            response = requests.get(pdf_url, stream=True)
+            print(f"  DEBUG: HTTP Status Code for {pdf_url}: {response.status_code}") # NEW DEBUG PRINT
+            response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
+            
+            pdf_data = io.BytesIO(response.content)
+            print(f"  DEBUG: BytesIO content length for {pdf_url}: {pdf_data.getbuffer().nbytes} bytes") # NEW DEBUG PRINT
+            
+            if pdf_data.getbuffer().nbytes == 0:
+                raise ValueError("Downloaded PDF content is empty.") # Raise error if content is empty
+
+            temp_pdf_path = f"/tmp/{uuid.uuid4().hex}.pdf"
+            # It's better to ensure the directory exists, although /tmp usually does
+            os.makedirs(os.path.dirname(temp_pdf_path), exist_ok=True) # Ensure /tmp directory exists
+            
+            with open(temp_pdf_path, 'wb') as f:
+                f.write(pdf_data.getvalue())
+            print(f"  DEBUG: Temporary PDF saved to: {temp_pdf_path}") # NEW DEBUG PRINT
+            
+            extracted_text = extract_text_from_pdf(temp_pdf_path)
+            os.remove(temp_pdf_path) # Clean up the temporary file after extraction
+        
+        except requests.exceptions.RequestException as e:
+            print(f"Error downloading PDF from {pdf_url}: {e}")
+            return
+        except ValueError as e: # Catch the new ValueError for empty content
+            print(f"Error processing downloaded PDF {pdf_url}: {e}")
+            return
+        except Exception as e: # Catch any other unexpected errors during file ops or extraction
+            print(f"Error processing downloaded PDF {pdf_url}: {e}")
+            return
+
+
+        if not extracted_text:
+            print(f"Warning: No text extracted from {display_name} ({pdf_url}). Skipping.")
+            return
+
+        chunks = chunk_text(extracted_text, self.chunk_size, self.overlap)
+        if not chunks:
+            print(f"Warning: No chunks generated for {display_name} ({pdf_url}). Skipping.")
+            return
+
+        documents_to_add = []
+        metadatas_to_add = []
+        ids_to_add = []
+
+        print(f"  Generating embeddings for {len(chunks)} chunks and preparing for ChromaDB: {display_name}...")
+        
+        encoded_results = self.embedding_model.encode(
+            chunks, 
+            batch_size=32,
+            return_dense=True, 
+            return_sparse=False, 
+            return_colbert_vecs=False
+        )
+        chunk_embeddings = encoded_results["dense_vecs"]
+
+        for i, chunk in enumerate(chunks):
+            unique_id = self._generate_chunk_id(pdf_url, i)
+            documents_to_add.append(chunk)
+            metadatas_to_add.append({"source": pdf_url, "display_name": display_name, "chunk_id": i})
+            ids_to_add.append(unique_id)
+        
+        self.collection.add(
+            documents=documents_to_add,
+            embeddings=chunk_embeddings.tolist(),
+            metadatas=metadatas_to_add,
+            ids=ids_to_add
+        )
+        
+        print(f"  {len(documents_to_add)} chunks from '{display_name}' added to ChromaDB.")
+        print(f"  Total chunks in collection: {self.collection.count()}")
+
+    def retrieve_context(self, query: str, top_k: int = 3) -> list[dict]:
+        """
+        Retrieves top_k most relevant document chunks for a given query from ChromaDB.
+        Returns a list of dictionaries, each containing 'text' and 'source' (URL or display name).
+        """
+        if self.collection.count() == 0:
+            print("Error: No documents indexed in ChromaDB. Cannot retrieve context.")
+            return []
+
+        print(f"Retrieving context for query: '{query}'")
+        
+        query_embedding_result = self.embedding_model.encode(
+            [query], 
+            batch_size=1,
+            return_dense=True, 
+            return_sparse=False, 
+            return_colbert_vecs=False
+        )
+        query_embedding = query_embedding_result["dense_vecs"].tolist()
+
+        results = self.collection.query(
+            query_embeddings=query_embedding,
+            n_results=top_k,
+            include=['documents', 'distances', 'metadatas']
+        )
+        
+        retrieved_chunks_info = []
+        if results and results['documents']:
+            for i, doc_text in enumerate(results['documents'][0]):
+                source_url = results['metadatas'][0][i].get('source', 'Unknown URL')
+                display_name = results['metadatas'][0][i].get('display_name', os.path.basename(urllib.parse.urlparse(source_url).path))
+                chunk_id_info = results['metadatas'][0][i].get('chunk_id', 'N/A')
+                distance_info = results['distances'][0][i]
+
+                retrieved_chunks_info.append({
+                    "text": doc_text,
+                    "source_url": source_url,
+                    "display_name": display_name
+                })
+                print(f"  Retrieved chunk {i+1} (distance: {distance_info:.4f}) from '{display_name}' (chunk {chunk_id_info}).")
+        else:
+            print("  No relevant chunks found in ChromaDB.")
+            
+        return retrieved_chunks_info
+
+    def get_conversation_history(self, conversation_id: str) -> list[dict]:
+        """Loads chat history from Firestore for a given conversation ID."""
+        if FIRESTORE_DATABASE is None:
+            print("Firestore not initialized. Cannot load conversation history.")
+            return []
+        
+        doc_ref = FIRESTORE_DATABASE.collection('conversations').document(conversation_id)
+        doc = doc_ref.get()
+        if doc.exists:
+            history = doc.to_dict().get('messages', [])
+            print(f"Loaded history for {conversation_id}: {len(history)} messages.")
+            return history
+        print(f"No history found for conversation ID: {conversation_id}")
+        return []
+
+    def save_conversation_history(self, conversation_id: str, history: list[dict]):
+        """Saves chat history to Firestore for a given conversation ID."""
+        if FIRESTORE_DATABASE is None:
+            print("Firestore not initialized. Cannot save conversation history.")
+            return
+
+        doc_ref = FIRESTORE_DATABASE.collection('conversations').document(conversation_id)
+        doc_ref.set({'messages': history})
+        print(f"Saved history for {conversation_id}: {len(history)} messages.")
+
+    def truncate_history(self, messages: list[dict], max_tokens: int = LLM_HISTORY_MAX_TOKENS) -> list[dict]:
+        """
+        Truncates conversation history to fit within a max_tokens limit for the LLM.
+        This is a simplistic truncation and doesn't use a tokenizer for exact token count.
+        """
+        current_len = sum(len(m['content']) for m in messages)
+        while current_len > max_tokens and len(messages) > 1: # Keep at least 1 message
+            if messages[0]['role'] == 'system':
+                if len(messages) >= 3:
+                    removed_user_msg = messages.pop(1)
+                    removed_ai_msg = messages.pop(1)
+                    current_len -= (len(removed_user_msg['content']) + len(removed_ai_msg['content']))
+                else:
+                    break
+            else:
+                removed_user_msg = messages.pop(0)
+                removed_ai_msg = messages.pop(0)
+                current_len -= (len(removed_user_msg['content']) + len(removed_ai_msg['content']))
+        return messages
+
+
+    def answer_question(self, question: str, conversation_id: str = None) -> str:
+        """
+        Answers a question by retrieving context, and querying DeepSeek.
+        Now includes conversational memory and uses global configs.
+        """
+        # Note: Document indexing is handled at FastAPI app startup for persistence.
+
+        # Get relevant context from ChromaDB
+        context_chunks_info = self.retrieve_context(question)
+        
+        context_parts = []
+        citation_info = {}
+
+        for chunk_info in context_chunks_info:
+            context_parts.append(chunk_info["text"])
+            source_key = chunk_info.get("display_name", chunk_info["source_url"])
+            if source_key not in citation_info:
+                citation_info[source_key] = True
+
+        context = "\n\n".join(context_parts)
+
+        context_prompt = ""
+        if context:
+            context_prompt = f"Using the following context:\n\n{context}\n\n"
+        else:
+            print("Warning: No relevant context found. Answering based on general knowledge or indicating lack of information.")
+
+        # --- Handle Conversational Memory ---
+        messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+        if conversation_id:
+            history = self.get_conversation_history(conversation_id)
+            if history:
+                messages.extend(history)
+
+        # Add current context and question
+        messages.append({"role": "user", "content": f"{context_prompt}Question: {question}"})
+
+        # Truncate conversation history if it's too long
+        messages = self.truncate_history(messages)
+
+        # Call DeepSeek API via OpenRouter
+        print("\nSending request to DeepSeek API...")
+        data = {
+            "model": "deepseek/deepseek-chat:free",
+            "messages": messages,
+            "temperature": LLM_TEMPERATURE,
+            "max_tokens": LLM_MAX_TOKENS,
+        }
+
+        response = requests.post(DEEPSEEK_API_URL, json=data, headers=DEEPSEEK_HEADERS)
+
+        if response.status_code == 200:
+            ai_response = response.json()
+            answer = ai_response['choices'][0]['message']['content']
+            print("\nDeepSeek Response:")
+            print(answer)
+
+            if citation_info:
+                citations_str = "\n\n**Sources:**\n" + "\n".join([f"- {name}" for name in citation_info.keys()])
+                answer += citations_str
+
+            if conversation_id:
+                messages.append({"role": "assistant", "content": answer})
+                self.save_conversation_history(conversation_id, messages)
+            
+            return answer
+        else:
+            error_message = f"Failed to fetch data from DeepSeek API. Status Code: {response.status_code}. Response: {response.text}"
+            print(error_message)
+            return f"Error: Could not get an answer from the AI. Details: {error_message}"
+
+# --- Main execution logic for local testing ---
+if __name__ == "__main__":
+    from dotenv import load_dotenv # Import load_dotenv for local execution
+    # Load environment variables from .env.local in the project root
+    load_dotenv(dotenv_path=os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.local'))
+
+    # Retrieve FIREBASE_CONFIG_BASE64 after loading dotenv
+    from config import FIREBASE_CONFIG_BASE64 
+
+    initialize_firebase_client()
+
+    rag_system = DocumentRAG(
+        embedding_model=embedding_model,
+        persist_directory=CHROMADB_PERSIST_DIRECTORY,
+        collection_name=CHROMADB_COLLECTION_NAME
+    )
+
+    print("\n--- Indexing Documents ---")
+    if FIRESTORE_DATABASE:
+        try:
+            docs_ref = FIRESTORE_DATABASE.collection('documents').stream()
+            firestore_pdf_infos = []
+            documents_processed_count = 0 # Track total documents found in Firestore
+            documents_skipped_non_pdf_count = 0 # Track documents skipped due to non-PDF URL
+
+            for doc in docs_ref:
+                documents_processed_count += 1
+                doc_data = doc.to_dict()
+                print(f"  DEBUG: Processing document ID: {doc.id}, Data: {doc_data}")
+                
+                if 'fileUrl' in doc_data:
+                    pdf_url = doc_data['fileUrl']
+                    print(f"  DEBUG: Found 'fileUrl': {pdf_url}")
+                    
+                    # Parse the URL to get the path part (without query parameters)
+                    parsed_url = urllib.parse.urlparse(pdf_url)
+                    file_path = parsed_url.path
+                    
+                    # Extract the filename from the path and normalize it
+                    file_name = os.path.basename(file_path)
+                    
+                    # DEBUGGING: Print the extracted file_name and the result of the check
+                    print(f"  DEBUG: Extracted file_name: '{file_name}'")
+                    is_pdf_check = isinstance(file_name, str) and file_name.strip().lower().endswith('.pdf')
+                    print(f"  DEBUG: is_pdf_check result: {is_pdf_check}")
+
+                    if is_pdf_check:
+                        display_name = doc_data.get('name_en', file_name) # Use file_name if name_en is missing
+                        firestore_pdf_infos.append({"url": pdf_url, "name": display_name})
+                        print(f"Found PDF in Firestore: {display_name} ({pdf_url}) - Qualified for indexing.")
+                    else:
+                        documents_skipped_non_pdf_count += 1
+                        # Corrected debug print for non-PDFs to show the file_name being evaluated
+                        print(f"  DEBUG: Skipped: '{file_name}' (Type: {type(file_name)}) does not end with '.pdf' (case-insensitive, stripped). Original URL: '{pdf_url}'")
+                else:
+                    documents_skipped_non_pdf_count += 1
+                    print(f"  DEBUG: Document ID: {doc.id} does not contain 'fileUrl'. Document data: {doc_data}")
+
+            if documents_processed_count == 0:
+                print("No documents found in Firestore collection 'documents' via stream(). Please check collection name and security rules.")
+            elif documents_processed_count > 0 and not firestore_pdf_infos:
+                print(f"Found {documents_processed_count} documents in Firestore, but none matched the '.pdf' criteria (all {documents_skipped_non_pdf_count} documents skipped).")
+            elif documents_skipped_non_pdf_count > 0:
+                print(f"Found {documents_processed_count} documents in Firestore. {len(firestore_pdf_infos)} PDFs qualified, {documents_skipped_non_pdf_count} documents skipped (non-PDF or missing fileUrl).")
+
+
+            if firestore_pdf_infos:
+                for pdf_info in firestore_pdf_infos:
+                    rag_system.add_document(pdf_info['url'], pdf_info['name'])
+            else:
+                pass # Specific messages already printed above.
+                
+        except Exception as e:
+            print(f"Error fetching documents from Firestore: {e}")
+            print("Please ensure your Firestore database is accessible and the service account key is correct.")
+    else:
+        print("Firestore client not initialized. Cannot fetch documents from Firestore.")
+        print("Using local PDF_DOCUMENT_PATHS as a fallback for testing purposes (ensure these files exist).")
+        # This import is moved here to avoid circular dependency if config imports rag_system
+        from .config import PDF_DOCUMENT_PATHS
+        for pdf_path in PDF_DOCUMENT_PATHS:
+            if os.path.exists(pdf_path):
+                rag_system.add_document(pdf_path)
+            else:
+                print(f"Error: Local PDF file not found at {pdf_path}. Skipping.")
+
+
+    print("\n--- Chat With CompassIA (Type 'q' to exit) ---")
+    current_conversation_id = str(uuid.uuid4())
+    print(f"Starting new local conversation with ID: {current_conversation_id}")
+
+    while True:
+        user_question = input("\nHow can I help you? ")
+        if user_question.lower() == 'q':
+            print("Exiting chat.")
+            break
+        
+        rag_system.answer_question(user_question, conversation_id=current_conversation_id)

src/config.py

@@ -0,0 +1,59 @@
+import os
+
+# --- OpenRouter DeepSeek API Configuration ---
+# Your DeepSeek API key, fetched from environment variables.
+# This should be set as a secret on Hugging Face Spaces.
+DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_R1_V3_API_KEY")
+if DEEPSEEK_API_KEY:
+    DEEPSEEK_API_KEY = DEEPSEEK_API_KEY.strip()
+
+# Base URL for the OpenRouter API.
+DEEPSEEK_API_URL = 'https://openrouter.ai/api/v1/chat/completions'
+
+# Headers required for OpenRouter API authentication.
+DEEPSEEK_HEADERS = {
+    'Authorization': f'Bearer {DEEPSEEK_API_KEY}',
+    'Content-Type': 'application/json'
+}
+
+# --- Embedding Model Configuration ---
+# Name of the Hugging Face model for embeddings.
+EMBEDDING_MODEL_NAME = 'BAAI/bge-m3'
+# Use float16 for reduced memory usage if supported by hardware (e.g., GPU).
+# Set to False if encountering issues on CPU.
+EMBEDDING_MODEL_USE_FP16 = True 
+
+# --- ChromaDB Configuration ---
+# Directory where ChromaDB will persist its database files.
+# This should be relative to your application's working directory.
+CHROMADB_PERSIST_DIRECTORY = "./chroma_db"
+# Name of the collection within ChromaDB where document chunks will be stored.
+CHROMADB_COLLECTION_NAME = "pdf_documents_collection"
+
+# --- Document Chunking Configuration ---
+# Maximum size of text chunks for embedding and retrieval.
+CHUNK_SIZE = 700
+# Overlap between consecutive chunks to maintain context.
+CHUNK_OVERLAP = 100
+
+# --- LLM Response Parameters ---
+# Temperature for the DeepSeek model. Lower values make output more deterministic.
+LLM_TEMPERATURE = 0.5
+# Maximum number of tokens the LLM can generate in a response.
+LLM_MAX_TOKENS = 500
+# Max tokens for conversation history truncation (approximate, not exact token count)
+LLM_HISTORY_MAX_TOKENS = 3000
+
+# --- Tesseract and Poppler Configuration (Docker/Deployment Specific) ---
+# Environment variables set in Dockerfile for Tesseract.
+TESSDATA_PREFIX = os.getenv("TESSDATA_PREFIX", "/usr/share/tesseract-ocr/4.00/tessdata")
+TESSERACT_CMD = os.getenv("TESSERACT_CMD", "/usr/bin/tesseract")
+# Path to Poppler's bin directory if not in system PATH (mostly for local Windows setup).
+POPPLER_PATH = None # e.g., r'C:\path\to\poppler\bin'
+
+
+
+# --- Firebase Configuration (for Conversational Memory) ---
+# Base64 encoded JSON string of your Firebase Service Account Key.
+# This should be set as a secret on Hugging Face Spaces.
+FIREBASE_CONFIG_BASE64 = os.getenv("FIREBASE_CONFIG_BASE64")

src/pdf_processing.py

@@ -0,0 +1,136 @@
+import os
+import io
+import re
+from PIL import Image
+
+# For text extraction from PDFs (non-OCR)
+from pdfminer.high_level import extract_text_to_fp
+from pdfminer.layout import LAParams
+
+# For image-based PDFs (OCR)
+from pdf2image import convert_from_path
+import pytesseract
+
+# Import Tesseract configuration from config.py
+from config import TESSDATA_PREFIX, TESSERACT_CMD, POPPLER_PATH
+
+# Set Tesseract command explicitly (uses ENV from Dockerfile or default)
+pytesseract.pytesseract.tesseract_cmd = TESSERACT_CMD
+if POPPLER_PATH:
+    # This setting is usually only needed for local Windows development
+    # where Poppler isn't in system PATH.
+    # In Docker, Poppler should be in PATH via apt-get install.
+    pass # No direct setting in pdf2image, but convert_from_path can accept poppler_path
+
+
+def extract_text_from_pdf(pdf_path: str) -> str:
+    """
+    Extracts text from a PDF. Tries direct text extraction first.
+    If sparse text is found (suggesting image-based PDF), it performs OCR.
+    """
+    print(f"Attempting direct text extraction from: {pdf_path}")
+    output_string = io.StringIO()
+    with open(pdf_path, 'rb') as fp:
+        try:
+            extract_text_to_fp(fp, output_string, laparams=LAParams())
+            text = output_string.getvalue()
+            # If text is very short for a non-empty PDF, it might be image-based.
+            # Using a threshold of 100 characters for extracted text and file size > 10KB.
+            if len(text.strip()) < 100 and os.path.getsize(pdf_path) > 10000:
+                print("Direct extraction yielded sparse text. Attempting OCR...")
+                return ocr_pdf(pdf_path)
+            return text
+        except Exception as e:
+            print(f"Direct PDF text extraction failed ({e}). Attempting OCR...")
+            return ocr_pdf(pdf_path)
+
+def ocr_pdf(pdf_path: str) -> str:
+    """
+    Performs OCR on a PDF file using pdf2image and pytesseract.
+    Requires Tesseract and Poppler to be installed and in system PATH.
+    """
+    all_text = []
+    try:
+        # Convert PDF pages to images. Higher DPI for better OCR.
+        # Pass poppler_path=POPPLER_PATH if it's set for local dev (Docker handles it via PATH)
+        images = convert_from_path(pdf_path, dpi=300) 
+
+        print(f"  Performing OCR on {len(images)} pages...")
+        for i, img in enumerate(images):
+            # Tesseract language packs: 'eng' for English, 'tur' for Turkish
+            # Dockerfile should install 'tesseract-ocr-ara' and 'tesseract-ocr-fra'
+            # if you need Arabic and French OCR.
+            page_text = pytesseract.image_to_string(img, lang='eng+tur+ara+fra') # Updated languages
+            all_text.append(page_text)
+            print(f"    Page {i+1} OCR complete.")
+
+    except Exception as e:
+        print(f"OCR process failed: {e}")
+        print("Please ensure Tesseract OCR and Poppler are correctly installed and their executables are in your system's PATH.")
+        return ""
+
+    return "\n".join(all_text)
+
+def chunk_text(text: str, max_chunk_size: int = 700, overlap: int = 100) -> list[str]:
+    """
+    Splits text into chunks of a maximum size with optional overlap.
+    Aims to split by paragraphs/sentences first, then by word.
+    """
+    if not text:
+        return []
+
+    # Simple paragraph-based chunking
+    paragraphs = re.split(r'\n\s*\n', text)
+    chunks = []
+    current_chunk = []
+    current_chunk_len = 0
+
+    for para in paragraphs:
+        if not para.strip():
+            continue
+
+        # If adding paragraph plus a separator exceeds max_chunk_size,
+        # or if the current_chunk is already substantial and adding this makes it too big,
+        # then finalize the current chunk.
+        if current_chunk_len + len(para) + len('\n\n') > max_chunk_size:
+            if current_chunk: # Only append if current_chunk is not empty
+                chunks.append("\n\n".join(current_chunk))
+                current_chunk = []
+                current_chunk_len = 0
+
+            # If a single paragraph is larger than max_chunk_size, split it by words
+            if len(para) > max_chunk_size:
+                words = para.split(' ')
+                sub_chunk = []
+                sub_chunk_len = 0
+                for word in words:
+                    if sub_chunk_len + len(word) + len(' ') > max_chunk_size:
+                        chunks.append(" ".join(sub_chunk))
+                        sub_chunk = [word]
+                        sub_chunk_len = len(word)
+                    else:
+                        sub_chunk.append(word)
+                        sub_chunk_len += len(word) + len(' ')
+                if sub_chunk: # Add remaining sub-chunk
+                    chunks.append(" ".join(sub_chunk))
+            else: # Paragraph fits into a new chunk
+                current_chunk.append(para)
+                current_chunk_len += len(para) + len('\n\n')
+        else: # Paragraph fits into the current chunk
+            current_chunk.append(para)
+            current_chunk_len += len(para) + len('\n\n')
+
+    if current_chunk: # Add any remaining text
+        chunks.append("\n\n".join(current_chunk))
+    
+    # Apply overlap: This is a simplistic overlap implementation.
+    final_chunks_with_overlap = []
+    for i in range(len(chunks)):
+        chunk = chunks[i]
+        if i > 0 and overlap > 0:
+            # Take a portion of the previous chunk to overlap
+            prev_chunk_part = chunks[i-1][-overlap:]
+            chunk = prev_chunk_part + "\n" + chunk
+        final_chunks_with_overlap.append(chunk)
+
+    return final_chunks_with_overlap

src/prompt.py

@@ -0,0 +1,20 @@
+SYSTEM_PROMPT = """
+You are CompassIA, the intelligent assistant for MaarifCompass, committed to supporting Turkiye Maarif Foundation graduates residing in Turkiye.
+
+Your core function is to deliver precise, document-backed information concerning their needs, primarily focusing on:
+- University application procedures, requirements, tuition fees, and scholarship opportunities
+- Accommodation and housing resources
+- Career networking and professional development
+- Relevant administrative and support services
+- Information related to Turkiye Maarif Foundation, Turkiye Scholarship, and more.
+
+You operate exclusively with data from a designated Document Center. **All information you provide must be directly sourced and verifiable from these internal documents.**
+
+**If a query falls outside the scope of the provided documents or lacks a direct answer within them, you must state that the information is not available in your current knowledge base. Do not offer external insights, assumptions, or speculate. This is a critical constraint.**
+
+Always cite the specific document name and page number for every piece of information provided. If information is aggregated from multiple sources, cite all relevant documents and page numbers.
+
+Your answers must be highly accurate, directly relevant, and easy to understand. Prioritize the user's query strictly based on documented facts.
+
+**Remember, always respond to the user in the language of their question.**
+"""