Added prompt system to compassia

.gitignore

@@ -6,3 +6,4 @@ all-libraries.txt
 # Ignore ChromaDB persistent storage
 chroma_db/
 
+temp*

compassia.py

@@ -1,10 +1,3 @@
-import requests
-import os
-import io
-import re
-import uuid # For generating unique IDs for ChromaDB
-from PIL import Image
-
 import sys
 # IMPORTANT: These lines MUST be at the very top of compassia.py
 # They ensure that any subsequent import of 'sqlite3' (even indirectly by chromadb)
@@ -15,6 +8,12 @@ try:
 except ImportError:
     pass # Fallback if pysqlite3 isn't available, but it should be in Docker
 
+import requests
+import os
+import io
+import re
+import uuid # For generating unique IDs for ChromaDB
+from PIL import Image
 
 # For text extraction from PDFs (non-OCR)
 from pdfminer.high_level import extract_text_to_fp
@@ -25,14 +24,14 @@ from pdf2image import convert_from_path
 import pytesseract
 
 # For embeddings and vector search
-from FlagEmbedding import BGEM3FlagModel # Using BGEM3FlagModel directly as per your latest code
+from FlagEmbedding import BGEM3FlagModel
 import chromadb # pip install chromadb
 
 # --- IMPORTANT: Configure Paths for Tesseract and Poppler ---
-# If Tesseract is not in your system's PATH, uncomment and set this:
+# If Tesseract is not in your system's PATH, uncomment and set this locally:
 # pytesseract.pytesseract.tesseract_cmd = r'C:\Program Files\Tesseract-OCR\tesseract.exe'
 
-# If pdf2image gives errors about poppler, uncomment and set this:
+# If pdf2image gives errors about poppler, uncomment and set this locally:
 # poppler_path = r'C:\path\to\poppler\bin'
 
 # --- OpenRouter DeepSeek API Configuration ---
@@ -50,21 +49,15 @@ HEADERS = {
 }
 
 # --- Embedding Model Configuration (Local BGE-M3) ---
-# IMPORTANT: This assumes you've run 'pip install -U FlagEmbedding'
-# BGE-M3 is multilingual, which is good for Turkish PDFs.
-# You might need to download the model weights the first time it's initialized.
-# Ensure you have enough RAM/VRAM for the model.
 print("Loading FlagEmbedding (BGE-M3) model...")
 try:
-    # Initialize BGEM3FlagModel. It will download weights to Hugging Face cache
-    # the first time, hence the need for disk space.
     embedding_model = BGEM3FlagModel('BAAI/bge-m3', use_fp16=True)
     print("FlagEmbedding (BGE-M3) model loaded successfully.")
 except Exception as e:
     print(f"Error loading FlagEmbedding model: {e}")
     print("Ensure you have resolved disk space issues for model download and have enough memory.")
     print("You might need to adjust 'use_fp16' based on your hardware (e.g., False for CPU/older GPUs).")
-    exit(1) # Exit if embedding model fails to load
+    exit(1)
 
 # --- PDF Processing Functions ---
 
@@ -77,11 +70,9 @@ def extract_text_from_pdf(pdf_path: str) -> str:
     output_string = io.StringIO()
     with open(pdf_path, 'rb') as fp:
         try:
-            # Use LAParams for better layout analysis
             extract_text_to_fp(fp, output_string, laparams=LAParams())
             text = output_string.getvalue()
-            # Basic check: if text is very short for a non-empty PDF, it might be image-based
-            if len(text.strip()) < 100 and os.path.getsize(pdf_path) > 10000: # Check file size as well
+            if len(text.strip()) < 100 and os.path.getsize(pdf_path) > 10000:
                 print("Direct extraction yielded sparse text. Attempting OCR...")
                 return ocr_pdf(pdf_path)
             return text
@@ -96,17 +87,15 @@ def ocr_pdf(pdf_path: str) -> str:
     """
     all_text = []
     try:
-        # Convert PDF pages to images. Higher DPI for better OCR.
-        # Pass poppler_path=poppler_path if it's not in your system's PATH
-        images = convert_from_path(pdf_path, dpi=300) # You can adjust dpi for quality vs. speed
+        images = convert_from_path(pdf_path, dpi=300)
 
         print(f"  Performing OCR on {len(images)} pages...")
         for i, img in enumerate(images):
-            # Optional: Basic image preprocessing for better OCR
-            # img = img.convert('L') # Convert to grayscale
-            # img = img.point(lambda x: 0 if x < 128 else 255, '1') # Binarize
-
-            # Perform OCR (lang='eng+tur' for English and Turkish support)
+            # Tesseract language packs:
+            # 'eng' for English, 'tur' for Turkish
+            # If you have scanned PDFs in Arabic or French, you MUST install
+            # 'tesseract-ocr-ara' and 'tesseract-ocr-fra' in your Dockerfile
+            # and change 'lang' to 'eng+tur+ara+fra'.
             page_text = pytesseract.image_to_string(img, lang='eng+tur')
             all_text.append(page_text)
             print(f"    Page {i+1} OCR complete.")
@@ -114,7 +103,7 @@ def ocr_pdf(pdf_path: str) -> str:
     except Exception as e:
         print(f"OCR process failed: {e}")
         print("Please ensure Tesseract OCR and Poppler are correctly installed and their executables are in your system's PATH.")
-        return "" # Return empty string if OCR fails
+        return ""
 
     return "\n".join(all_text)
 
@@ -122,12 +111,10 @@ def chunk_text(text: str, max_chunk_size: int = 700, overlap: int = 100) -> list
     """
     Splits text into chunks of a maximum size with optional overlap.
     Aims to split by paragraphs/sentences first, then by word.
-    Note: Increased max_chunk_size to 700 to match your previous code's `chunk_size` for RAG.
     """
     if not text:
         return []
 
-    # Simple paragraph-based chunking
     paragraphs = re.split(r'\n\s*\n', text)
     chunks = []
     current_chunk = []
@@ -137,16 +124,12 @@ def chunk_text(text: str, max_chunk_size: int = 700, overlap: int = 100) -> list
         if not para.strip():
             continue
 
-        # If adding paragraph plus a separator exceeds max_chunk_size,
-        # or if the current_chunk is already substantial and adding this makes it too big,
-        # then finalize the current chunk.
         if current_chunk_len + len(para) + len('\n\n') > max_chunk_size:
-            if current_chunk: # Only append if current_chunk is not empty
+            if current_chunk:
                 chunks.append("\n\n".join(current_chunk))
                 current_chunk = []
                 current_chunk_len = 0
 
-            # If a single paragraph is larger than max_chunk_size, split it by words
             if len(para) > max_chunk_size:
                 words = para.split(' ')
                 sub_chunk = []
@@ -159,32 +142,28 @@ def chunk_text(text: str, max_chunk_size: int = 700, overlap: int = 100) -> list
                     else:
                         sub_chunk.append(word)
                         sub_chunk_len += len(word) + len(' ')
-                if sub_chunk: # Add remaining sub-chunk
+                if sub_chunk:
                     chunks.append(" ".join(sub_chunk))
-            else: # Paragraph fits into a new chunk
+            else:
                 current_chunk.append(para)
                 current_chunk_len += len(para) + len('\n\n')
-        else: # Paragraph fits into the current chunk
+        else:
             current_chunk.append(para)
             current_chunk_len += len(para) + len('\n\n')
 
-    if current_chunk: # Add any remaining text
+    if current_chunk:
         chunks.append("\n\n".join(current_chunk))
-    
-    # Apply overlap: This is a simplistic overlap implementation.
-    # For more robust RAG, consider sentence-window retrieval or more advanced chunking libraries.
+
     final_chunks_with_overlap = []
     for i in range(len(chunks)):
         chunk = chunks[i]
         if i > 0 and overlap > 0:
-            # Take a portion of the previous chunk to overlap
             prev_chunk_part = chunks[i-1][-overlap:]
             chunk = prev_chunk_part + "\n" + chunk
         final_chunks_with_overlap.append(chunk)
 
     return final_chunks_with_overlap
 
-
 # --- RAG Core Functions with ChromaDB ---
 
 class DocumentRAG:
@@ -195,30 +174,21 @@ class DocumentRAG:
         self.persist_directory = persist_directory
         self.collection_name = collection_name
         
-        # Initialize ChromaDB client and collection
         print(f"Initializing ChromaDB at: {self.persist_directory}")
         self.client = chromadb.PersistentClient(path=self.persist_directory)
         
-        # Get or create the collection
         self.collection = self.client.get_or_create_collection(
             name=self.collection_name,
-            # Genkit uses 'cosine' by default. 'l2' (Euclidean) or 'ip' (Inner Product)
-            # are also common. BGE-M3 generally uses cosine.
             metadata={"hnsw:space": "cosine"}
         )
         print(f"ChromaDB collection '{self.collection_name}' ready.")
 
     def _generate_chunk_id(self, pdf_path: str, chunk_idx: int) -> str:
-        """Generates a unique ID for each chunk based on file path and index."""
-        # Use UUID to ensure uniqueness even if paths are similar or contain problematic chars
         return f"{os.path.basename(pdf_path)}_{chunk_idx}_{uuid.uuid4().hex}"
 
     def add_document(self, pdf_path: str):
         print(f"Adding document: {pdf_path}")
         
-        # Check if the document has already been indexed in ChromaDB
-        # We'll use the file path as a simple way to check if _any_ chunk from this PDF exists.
-        # A more robust check might involve hashing the file content or checking specific metadata.
         results = self.collection.get(
             where={"source": pdf_path},
             limit=1
@@ -237,43 +207,31 @@ class DocumentRAG:
             print(f"Warning: No chunks generated for {pdf_path}. Skipping.")
             return
 
-        # Prepare data for ChromaDB
         documents_to_add = []
         metadatas_to_add = []
         ids_to_add = []
 
         print(f"  Generating embeddings for {len(chunks)} chunks and preparing for ChromaDB...")
         
-        # BGE-M3's encode method returns a dictionary for dense, sparse, etc.
-        # We need the 'dense_vecs' for standard vector search.
         encoded_results = self.embedding_model.encode(
             chunks, 
-            batch_size=32, # Adjust batch_size if out of memory
+            batch_size=32,
             return_dense=True, 
             return_sparse=False, 
             return_colbert_vecs=False
         )
         
-        # Extract only the dense vectors for ChromaDB
         chunk_embeddings = encoded_results["dense_vecs"]
 
-        # Ensure embeddings are normalized if using cosine similarity with IP index,
-        # but ChromaDB's 'cosine' space handles this internally.
-        # If using FAISS with IP, you'd normalize here:
-        # from numpy.linalg import norm
-        # chunk_embeddings = chunk_embeddings / norm(chunk_embeddings, axis=1, keepdims=True)
-
-
         for i, chunk in enumerate(chunks):
             unique_id = self._generate_chunk_id(pdf_path, i)
             documents_to_add.append(chunk)
             metadatas_to_add.append({"source": pdf_path, "chunk_id": i})
             ids_to_add.append(unique_id)
         
-        # Add to ChromaDB collection
         self.collection.add(
             documents=documents_to_add,
-            embeddings=chunk_embeddings.tolist(), # Convert numpy array to list of lists
+            embeddings=chunk_embeddings.tolist(),
             metadatas=metadatas_to_add,
             ids=ids_to_add
         )
@@ -291,7 +249,6 @@ class DocumentRAG:
 
         print(f"Retrieving context for query: '{query}'")
         
-        # Encode the query using the embedding model
         query_embedding_result = self.embedding_model.encode(
             [query], 
             batch_size=1,
@@ -299,9 +256,8 @@ class DocumentRAG:
             return_sparse=False, 
             return_colbert_vecs=False
         )
-        query_embedding = query_embedding_result["dense_vecs"].tolist() # Get dense vector and convert to list
+        query_embedding = query_embedding_result["dense_vecs"].tolist()
 
-        # Query ChromaDB
         results = self.collection.query(
             query_embeddings=query_embedding,
             n_results=top_k,
@@ -313,7 +269,7 @@ class DocumentRAG:
             for i, doc_text in enumerate(results['documents'][0]):
                 source_info = results['metadatas'][0][i].get('source', 'Unknown Source')
                 chunk_id_info = results['metadatas'][0][i].get('chunk_id', 'N/A')
-                distance_info = results['distances'][0][i] # Smaller distance is more similar for cosine
+                distance_info = results['distances'][0][i]
 
                 retrieved_chunks_texts.append(doc_text)
                 print(f"  Retrieved chunk {i+1} (distance: {distance_info:.4f}) from '{source_info}' (chunk {chunk_id_info}).")
@@ -327,12 +283,9 @@ class DocumentRAG:
         Answers a question by ensuring PDFs are indexed, retrieving context,
         and querying DeepSeek.
         """
-        # Ensure documents are added for specified paths.
-        # This will now intelligently skip already indexed documents.
         for path in pdf_paths:
             self.add_document(path)
 
-        # Get relevant context from ChromaDB
         context_chunks = self.retrieve_context(question)
         context = "\n\n".join(context_chunks)
 
@@ -342,19 +295,36 @@ class DocumentRAG:
         else:
             context_prompt = f"Using the following context:\n\n{context}\n\n"
 
-        # Construct prompt for DeepSeek
+        # --- UPDATED SYSTEM PROMPT FOR COMPASSIA AI ---
+        system_prompt = """
+            You are CompassIA, the intelligent assistant for MaarifCompass, committed to supporting Turkiye Maarif Foundation graduates residing in Turkiye.
+
+            Your core function is to deliver precise, document-backed information concerning their needs, primarily focusing on:
+            - University application procedures, requirements, tuition fees, and scholarship opportunities
+            - Accommodation and housing resources
+            - Career networking and professional development
+            - Relevant administrative and support services.
+            - Information related to Turkiye Maarif Foundation, Turkiye Scholarship and more.
+
+            You operate exclusively with data from a designated Document Center. **It is imperative that every piece of information you provide is directly sourced and verifiable from these internal documents.**
+
+            **Should a query fall outside the scope of the provided documents or lack a direct answer within them, you are required to politely inform the user that the specific information is not available in your current knowledge base, without offering any external insights or assumptions.**
+
+            Your answers should be highly accurate, directly relevant, easy to understand, and always prioritize the user's query based strictly on documented facts.
+            **Remember, you always answer the user with the language of the question.**
+        """
+
         messages = [
-            {"role": "system", "content": "You are an AI assistant specialized in answering questions based on provided context. If the answer is not in the context, state that explicitly. If you cannot answer based *solely* on the context, politely indicate that the information is not available in the provided documents."},
+            {"role": "system", "content": system_prompt},
             {"role": "user", "content": f"{context_prompt}Question: {question}"}
         ]
 
-        # Call DeepSeek API via OpenRouter
         print("\nSending request to DeepSeek API...")
         data = {
-            "model": "deepseek/deepseek-chat:free", # Using the specified free model
+            "model": "deepseek/deepseek-chat:free",
             "messages": messages,
-            "temperature": 0.5, # Adjust for creativity vs. factualness
-            "max_tokens": 500, # Limit response length
+            "temperature": 0.5,
+            "max_tokens": 500,
         }
 
         response = requests.post(API_URL, json=data, headers=HEADERS)
@@ -372,30 +342,22 @@ class DocumentRAG:
 
 # --- Main execution logic ---
 if __name__ == "__main__":
-    # Initialize the RAG system with ChromaDB persistence
-    # The 'chroma_db' directory will be created in your project root.
     rag_system = DocumentRAG(
         embedding_model=embedding_model,
-        persist_directory="./chroma_db", # This is where your vector DB will be saved
-        collection_name="pdf_documents_collection", # A unique name for your collection
+        persist_directory="./chroma_db",
+        collection_name="pdf_documents_collection",
         chunk_size=700,
         overlap=100
     )
 
-    # --- Define your PDF documents ---
-    # Replace with the actual paths to your PDF files.
-    # For testing, ensure 'documents' directory exists and contains your PDFs.
     pdf_document_paths = [
-        "documents/heracles_tr.pdf", # Heracles TR PDF path
-        "documents/heracles_en.pdf", # Heracles EN PDF path
-        # Add more PDF paths here if you have them
+        "documents/heracles_tr.pdf",
+        "documents/heracles_en.pdf",
         "documents/ogrenci_katki_payi_ogrenim_ucretleri.pdf",
         "documents/Ogrenci_Liderligi_Burs_Programi_Sozlesme_Metni_2024-2025.pdf",
         "documents/tmv-bursluluk-yonergesi.pdf"
     ]
 
-    # --- Add PDFs to the RAG system for indexing ---
-    # This will now process only new or unindexed documents.
     print("\n--- Indexing Documents ---")
     for pdf_path in pdf_document_paths:
         if os.path.exists(pdf_path):
@@ -403,13 +365,11 @@ if __name__ == "__main__":
         else:
             print(f"Error: PDF file not found at {pdf_path}. Please check the path.")
 
-    # --- Start Chat Loop ---
-    print("\n--- PDF Chat with DeepSeek (Type 'quit' to exit) ---")
+    print("\n--- Chat With CompassIA (Type 'quit' to exit) ---")
     while True:
-        user_question = input("\nYour question about the PDF(s): ")
+        user_question = input("\nHow can I help you? ")
         if user_question.lower() == 'quit':
             print("Exiting chat.")
             break
         
-        # No need to pass pdf_document_paths here; documents are already in ChromaDB
-        rag_system.answer_question(user_question, []) # Pass an empty list, as documents are in DB
+        rag_system.answer_question(user_question, pdf_document_paths)