Change to Chatbot

.chainlit/config.toml

@@ -1,8 +1,4 @@
 [project]
-# Whether to enable telemetry (default: false). No personal data is collected.
-enable_telemetry = false
-
-
 # List of environment variables to be provided by each user to use the app.
 user_env = []
 
@@ -34,6 +30,10 @@ auto_tag_thread = true
 # Allow users to edit their own messages
 edit_message = true
 
+[features.slack]
+# Add emoji reaction when message is received (requires reactions:write OAuth scope)
+reaction_on_message_received = false
+
 # Authorize users to spontaneously upload files with messages
 [features.spontaneous_file_upload]
     enabled = false
@@ -57,6 +57,9 @@ edit_message = true
 [features.mcp.sse]
     enabled = true
 
+[features.mcp.streamable-http]
+    enabled = true
+
 [features.mcp.stdio]
     enabled = true
     # Only the executables in the allow list can be used for MCP stdio server.
@@ -106,13 +109,13 @@ alert_style = "classic"
 
 
 # Specify a custom meta image url.
-# custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
 
 # Load assistant logo directly from URL.
-logo_file_url = ""
+logo_file_url = "public/favicon.svg"
 
 # Load assistant avatar image directly from URL.
-default_avatar_file_url = ""
+default_avatar_file_url = "public/favicon.svg"
 
 # Specify a custom build directory for the frontend.
 # This can be used to customize the frontend code.
@@ -127,4 +130,4 @@ default_avatar_file_url = ""
 #     url = "https://github.com/Chainlit/chainlit/issues"
 
 [meta]
-generated_by = "2.6.0"
+generated_by = "2.6.9"

.chainlit/old_config.toml

@@ -0,0 +1,130 @@
+[project]
+# Whether to enable telemetry (default: false). No personal data is collected.
+enable_telemetry = false
+
+
+# List of environment variables to be provided by each user to use the app.
+user_env = []
+
+# Duration (in seconds) during which the session is saved when the connection is lost
+session_timeout = 3600
+
+# Duration (in seconds) of the user session expiry
+user_session_timeout = 1296000  # 15 days
+
+# Enable third parties caching (e.g., LangChain cache)
+cache = false
+
+# Authorized origins
+allow_origins = ["*"]
+
+[features]
+# Process and display HTML in messages. This can be a security risk (see https://stackoverflow.com/questions/19603097/why-is-it-dangerous-to-render-user-generated-html-or-javascript)
+unsafe_allow_html = false
+
+# Process and display mathematical expressions. This can clash with "$" characters in messages.
+latex = false
+
+# Autoscroll new user messages at the top of the window
+user_message_autoscroll = true
+
+# Automatically tag threads with the current chat profile (if a chat profile is used)
+auto_tag_thread = true
+
+# Allow users to edit their own messages
+edit_message = true
+
+# Authorize users to spontaneously upload files with messages
+[features.spontaneous_file_upload]
+    enabled = false
+    # Define accepted file types using MIME types
+    # Examples:
+    # 1. For specific file types:
+    #    accept = ["image/jpeg", "image/png", "application/pdf"]
+    # 2. For all files of certain type:
+    #    accept = ["image/*", "audio/*", "video/*"]
+    # 3. For specific file extensions:
+    #    accept = { "application/octet-stream" = [".xyz", ".pdb"] }
+    # Note: Using "*/*" is not recommended as it may cause browser warnings
+    accept = ["*/*"]
+    max_files = 20
+    max_size_mb = 500
+
+[features.audio]
+    # Sample rate of the audio
+    sample_rate = 24000
+
+[features.mcp.sse]
+    enabled = true
+
+[features.mcp.stdio]
+    enabled = true
+    # Only the executables in the allow list can be used for MCP stdio server.
+    # Only need the base name of the executable, e.g. "npx", not "/usr/bin/npx".
+    # Please don't comment this line for now, we need it to parse the executable name.
+    allowed_executables = [ "npx", "uvx" ]
+
+[UI]
+# Name of the assistant.
+name = "Assistant"
+
+# default_theme = "dark"
+
+# layout = "wide"
+
+# default_sidebar_state = "open"
+
+# Description of the assistant. This is used for HTML tags.
+# description = ""
+
+# Chain of Thought (CoT) display mode. Can be "hidden", "tool_call" or "full".
+cot = "full"
+
+# Specify a CSS file that can be used to customize the user interface.
+# The CSS file can be served from the public directory or via an external link.
+# custom_css = "/public/test.css"
+
+# Specify additional attributes for a custom CSS file
+# custom_css_attributes = "media=\"print\""
+
+# Specify a JavaScript file that can be used to customize the user interface.
+# The JavaScript file can be served from the public directory.
+# custom_js = "/public/test.js"
+
+# The style of alert boxes. Can be "classic" or "modern".
+alert_style = "classic"
+
+# Specify additional attributes for custom JS file
+# custom_js_attributes = "async type = \"module\""
+
+# Custom login page image, relative to public directory or external URL
+# login_page_image = "/public/custom-background.jpg"
+
+# Custom login page image filter (Tailwind internal filters, no dark/light variants)
+login_page_image_filter = "brightness-50 grayscale"
+login_page_image_dark_filter = "contrast-200 blur-sm"
+
+
+# Specify a custom meta image url.
+custom_meta_image_url = "https://chainlit-cloud.s3.eu-west-3.amazonaws.com/logo/chainlit_banner.png"
+
+# Load assistant logo directly from URL.
+logo_file_url = "https://avatars.githubusercontent.com/u/128686189?s=200&v=4"
+
+# Load assistant avatar image directly from URL.
+default_avatar_file_url = "https://avatars.githubusercontent.com/u/128686189?s=200&v=4"
+
+# Specify a custom build directory for the frontend.
+# This can be used to customize the frontend code.
+# Be careful: If this is a relative path, it should not start with a slash.
+# custom_build = "./public/build"
+
+# Specify optional one or more custom links in the header.
+# [[UI.header_links]]
+#     name = "Issues"
+#     display_name = "Report Issue"
+#     icon_url = "https://avatars.githubusercontent.com/u/128686189?s=200&v=4"
+#     url = "https://github.com/Chainlit/chainlit/issues"
+
+[meta]
+generated_by = "2.6.0"

.gitignore

@@ -225,4 +225,6 @@ secrets/
 volumes/
 
 simple_analysis.py
-#  This file is used for simple analysis of the codebase, such as checking for unused imports or variables.
+#  This file is used for simple analysis of the codebase, such as checking for unused imports or variables.
+
+RAGAS_test_details/

old_prisma_compose.yml

@@ -0,0 +1,30 @@
+services:
+  postgres:
+    image: postgres:16
+    volumes:
+      - ./.data/postgres:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER:-root}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-root}
+      - POSTGRES_DB=${POSTGRES_DB:-postgres}
+    ports:
+      - ${POSTGRES_PORT:-5432}:5432
+  localstack:
+    image: localstack/localstack:latest
+    environment:
+      SERVICES: s3
+    ports:
+      - 4566:4566
+    volumes:
+      - ./localstack-script.sh:/etc/localstack/init/ready.d/script.sh
+      - "/var/run/docker.sock:/var/run/docker.sock"
+  # Uncomment below to simulate Azure Blob Storage (don't forget to run the init_azure_storage.py to create the container)
+  # azurite:
+  #   image: mcr.microsoft.com/azure-storage/azurite
+  #   ports:
+  #     - "10000:10000"  # Blob service
+  #     - "10001:10001"  # Queue service
+  #     - "10002:10002"  # Table service
+  #   volumes:
+  #     - ./.data/azurite:/data
+  #   command: "azurite --blobHost 0.0.0.0 --queueHost 0.0.0.0 --tableHost 0.0.0.0"

populate_db.py

@@ -1,94 +1,113 @@
-import time
 import os
+import time
 from pathlib import Path
+
 from dotenv import load_dotenv
-from unstructured.cleaners.core import clean_extra_whitespace, replace_unicode_quotes, clean_dashes, group_broken_paragraphs
-from langchain_unstructured import UnstructuredLoader
-from sentence_transformers import SentenceTransformer
-from pymilvus import MilvusClient, DataType
 from langchain_nebius import NebiusEmbeddings
+from langchain_unstructured import UnstructuredLoader
 from pydantic import SecretStr
-import os
+from pymilvus import MilvusClient, DataType
+from unstructured.cleaners.core import (
+    clean_extra_whitespace, 
+    replace_unicode_quotes
+)
 
-# Load environment variables from .env file
+# Load environment variables
 load_dotenv()
 
-# Initialize Milvus client and collection setup
+# Configuration constants
 MILVUS_URI = os.getenv("MILVUS_URI", "http://localhost:19530")
+COLLECTION_NAME = "my_rag_collection"
+DOCUMENT_DIR = "data/"
+EMBEDDING_DIMENSION = 4096
+TEXT_MAX_LENGTH = 65000
+CHUNK_SIZE = 100
+BATCH_SIZE = 5
+
+# Chunking configuration
+MAX_CHARACTERS = 1500
+COMBINE_TEXT_UNDER_N_CHARS = 200
+
+# Initialize clients
 milvus_client = MilvusClient(uri=MILVUS_URI)
-collection_name = "my_rag_collection"
 
-# Initialize embedding model
-# embedding_model = SentenceTransformer("BAAI/bge-small-en-v1.5")
-# embedding_model = SentenceTransformer("Qwen/Qwen3-Embedding-0.6B")
 embedding_model = NebiusEmbeddings(
     api_key=SecretStr(os.getenv("NEBIUS_API_KEY", os.getenv("OPENAI_API_KEY"))),
     model="Qwen/Qwen3-Embedding-8B",
     base_url="https://api.studio.nebius.ai/v1"
 )
 
+def clean_text(text):
+    """Simple text cleaning for educational documents."""
+    import re
+    
+    # Basic cleaning without problematic functions
+    text = clean_extra_whitespace(text)
+    text = replace_unicode_quotes(text)
+    
+    # Simple normalizations
+    text = re.sub(r'[\r\n]+', ' ', text)  # Convert newlines to spaces
+    text = re.sub(r'\s+', ' ', text)      # Multiple spaces to single space
+    
+    return text.strip()
 
 
-def emb_text(text):
-    """Generate embeddings for text using the sentence transformer model."""
+def generate_embedding(text):
+    """Generate embedding for a single text."""
     return embedding_model.embed_query(text)
-    # return embedding_model.encode([text], normalize_embeddings=True).tolist()[0]
 
-def emb_text_batch(texts):
-    """Generate embeddings for multiple texts in batch - more efficient."""
+
+def generate_embeddings_batch(texts):
+    """Generate embeddings for multiple texts efficiently."""
     return embedding_model.embed_documents(texts)
 
-def process_embeddings_in_batches(texts_to_embed, batch_size=50):
-    """Process embeddings in batches with error handling and fallback."""
+
+def process_embeddings_in_batches(texts, batch_size=BATCH_SIZE):
+    """Process embeddings in batches with error handling."""
     all_embeddings = []
+    total_batches = (len(texts) + batch_size - 1) // batch_size
     
-    print(f"Generating embeddings in batches of {batch_size}...")
+    print(f"Generating embeddings in {total_batches} batches of {batch_size}...")
     
-    for i in range(0, len(texts_to_embed), batch_size):
-        batch_texts = texts_to_embed[i:i + batch_size]
-        batch_end = min(i + batch_size, len(texts_to_embed))
+    for i in range(0, len(texts), batch_size):
+        batch_texts = texts[i:i + batch_size]
+        batch_num = i // batch_size + 1
         
-        print(f"Processing embedding batch {i//batch_size + 1}/{(len(texts_to_embed) + batch_size - 1)//batch_size} (documents {i+1}-{batch_end})")
+        print(f"Processing batch {batch_num}/{total_batches}")
         
         try:
-            batch_embeddings = emb_text_batch(batch_texts)
+            batch_embeddings = generate_embeddings_batch(batch_texts)
             all_embeddings.extend(batch_embeddings)
-            
-            # Add a small delay between batches to be respectful to the API
-            time.sleep(1.5)
+            time.sleep(1.5)  # API rate limiting
             
         except Exception as e:
-            print(f"Error processing batch {i//batch_size + 1}: {e}")
-            print("Falling back to individual processing for this batch...")
+            print(f"Batch {batch_num} failed: {e}. Processing individually...")
             
-            # Fallback to individual processing for this batch
             for j, text in enumerate(batch_texts):
                 try:
-                    embedding = emb_text(text)
+                    embedding = generate_embedding(text)
                     all_embeddings.append(embedding)
-                    print(f"  Individual embedding {i+j+1} completed")
-                    time.sleep(2)  # Longer delay for individual requests
+                    time.sleep(2)
                 except Exception as individual_error:
-                    print(f"  Failed to process document {i+j+1}: {individual_error}")
-                    # Use a zero vector as fallback
-                    all_embeddings.append([0.0] * 4096)
+                    print(f"Failed to process document {i+j+1}: {individual_error}")
+                    all_embeddings.append([0.0] * EMBEDDING_DIMENSION)
     
     return all_embeddings
 
 def create_collection():
-    """Create collection if it doesn't exist."""
-    if milvus_client.has_collection(collection_name):
-        milvus_client.load_collection(collection_name=collection_name)
+    """Create Milvus collection if it doesn't exist."""
+    if milvus_client.has_collection(COLLECTION_NAME):
+        milvus_client.load_collection(collection_name=COLLECTION_NAME)
         return
     
-    # Create Milvus collection schema
+    # Create collection schema
     schema = milvus_client.create_schema(auto_id=False, enable_dynamic_field=False)
     schema.add_field(field_name="id", datatype=DataType.INT64, is_primary=True)
-    schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=4096)  # Qwen/Qwen3-Embedding-8B dimension
-    schema.add_field(field_name="text", datatype=DataType.VARCHAR, max_length=65535)  # Maximum allowed for VARCHAR
+    schema.add_field(field_name="vector", datatype=DataType.FLOAT_VECTOR, dim=EMBEDDING_DIMENSION)
+    schema.add_field(field_name="text", datatype=DataType.VARCHAR, max_length=65535)
     schema.add_field(field_name="metadata", datatype=DataType.JSON)
 
-    # Create index for vector search
+    # Create vector index
     index_params = MilvusClient.prepare_index_params()
     index_params.add_index(
         field_name="vector",
@@ -98,143 +117,191 @@ def create_collection():
 
     # Create and load collection
     milvus_client.create_collection(
-        collection_name=collection_name,
+        collection_name=COLLECTION_NAME,
         schema=schema,
         index_params=index_params,
         consistency_level="Strong",
     )
-    milvus_client.load_collection(collection_name=collection_name)
+    milvus_client.load_collection(collection_name=COLLECTION_NAME)
+
+def load_documents():
+    """Load documents from the data directory."""
+    file_extensions = ["*.pdf", "*.docx", "*.html"]
+    file_paths = []
+
+    for ext in file_extensions:
+        file_paths.extend(Path(DOCUMENT_DIR).glob(ext))
+
+    file_paths = [str(file) for file in file_paths]
+    
+    loader = UnstructuredLoader(
+        file_paths, 
+        chunking_strategy="by_title",
+        include_orig_elements=False
+    )
+    
+    docs = loader.load()
+    print(f"Loaded {len(docs)} initial documents")
+    
+    # Apply additional cleaning and chunking
+    final_chunks = []
+    
+    for doc in docs:
+        # Clean text
+        cleaned_text = clean_text(doc.page_content)
+        
+        # Skip very short chunks
+        if len(cleaned_text) < 50:
+            continue
+            
+        # Split if too large
+        if len(cleaned_text) <= MAX_CHARACTERS:
+            doc.page_content = cleaned_text
+            final_chunks.append(doc)
+        else:
+            # Split large chunks on sentence boundaries
+            chunks = _split_large_chunk(cleaned_text, doc.metadata)
+            final_chunks.extend(chunks)
+    
+    print(f"Final processed chunks: {len(final_chunks)}")
+    if final_chunks:
+        avg_length = sum(len(doc.page_content) for doc in final_chunks) / len(final_chunks)
+        print(f"Average chunk length: {avg_length:.0f} characters")
+    
+    return final_chunks
+
+
+def _split_large_chunk(text, metadata):
+    """Split large text into smaller chunks."""
+    from langchain.schema import Document
+    
+    chunks = []
+    sentences = text.split('. ')
+    current_chunk = ""
+    
+    for sentence in sentences:
+        potential_chunk = current_chunk + sentence + '. '
+        
+        if len(potential_chunk) > MAX_CHARACTERS and len(current_chunk) > COMBINE_TEXT_UNDER_N_CHARS:
+            if current_chunk.strip():
+                chunks.append(Document(
+                    page_content=current_chunk.strip(),
+                    metadata=metadata.copy()
+                ))
+            current_chunk = sentence + '. '
+        else:
+            current_chunk = potential_chunk
+    
+    # Add remaining content
+    if current_chunk.strip():
+        chunks.append(Document(
+            page_content=current_chunk.strip(),
+            metadata=metadata.copy()
+        ))
+    
+    return chunks
+
+
+def prepare_document_data(docs, start_idx=0):
+    """Prepare document data for insertion."""
+    texts_to_embed = []
+    doc_data = []
+    
+    for i, doc in enumerate(docs):
+        text_content = doc.page_content
+        if len(text_content) > TEXT_MAX_LENGTH:
+            text_content = text_content[:TEXT_MAX_LENGTH]
+            print(f"Document {start_idx + i + 1} truncated to {TEXT_MAX_LENGTH} characters")
+        
+        texts_to_embed.append(text_content)
+        doc_data.append({
+            "id": start_idx + i,
+            "text": text_content,
+            "metadata": doc.metadata or {}
+        })
+    
+    return texts_to_embed, doc_data
+
 
-# Document directory
-directory_path = "data/"  
+def process_document_chunk(docs, chunk_idx, chunk_num, total_chunks):
+    """Process a single chunk of documents."""
+    print(f"\nProcessing chunk {chunk_num}/{total_chunks}")
+    
+    # Prepare document data
+    texts_to_embed, doc_data = prepare_document_data(docs, chunk_idx)
+    
+    # Generate embeddings
+    print(f"Generating embeddings for {len(texts_to_embed)} documents...")
+    embeddings = process_embeddings_in_batches(texts_to_embed)
+    
+    # Prepare data for insertion
+    data_to_insert = []
+    for doc_info, embedding in zip(doc_data, embeddings):
+        data_to_insert.append({
+            "id": doc_info["id"],
+            "vector": embedding,
+            "text": doc_info["text"],
+            "metadata": doc_info["metadata"]
+        })
+    
+    # Insert into Milvus
+    insert_result = milvus_client.insert(collection_name=COLLECTION_NAME, data=data_to_insert)
+    return insert_result['insert_count']
 
 def main():
-    """Main function to load documents and insert them into Milvus."""
+    """Main function to process and insert documents into Milvus."""
     create_collection()
     
     # Check if collection already has data
-    stats = milvus_client.get_collection_stats(collection_name)
+    stats = milvus_client.get_collection_stats(COLLECTION_NAME)
     if stats['row_count'] > 0:
         print(f"Collection already contains {stats['row_count']} documents. Skipping insertion.")
         return
     
-    docs = unstructured_document_loader()
+    # Load documents
+    docs = load_documents()
+    if not docs:
+        print("No documents found to process.")
+        return
     
-    # Process documents in small chunks to avoid memory issues on 4GB droplet
-    chunk_size = 100  # Very conservative chunk size for 4GB memory
+    # Process documents in chunks
     total_docs = len(docs)
-    total_chunks = (total_docs + chunk_size - 1) // chunk_size
-    
-    print(f"🔧 Memory-efficient processing: {total_docs} documents in {total_chunks} chunks of {chunk_size}")
-    print("📊 This approach prevents OOM kills on your 4GB DigitalOcean droplet")
-    
+    total_chunks = (total_docs + CHUNK_SIZE - 1) // CHUNK_SIZE
     total_inserted = 0
     
-    for chunk_idx in range(0, total_docs, chunk_size):
-        chunk_end = min(chunk_idx + chunk_size, total_docs)
-        chunk_num = chunk_idx // chunk_size + 1
-        
-        print(f"\n{'='*40}")
-        print(f"CHUNK {chunk_num}/{total_chunks} | Docs {chunk_idx + 1}-{chunk_end}")
-        print(f"{'='*40}")
-        
-        # Get current chunk of documents
+    print(f"Processing {total_docs} documents in {total_chunks} chunks of {CHUNK_SIZE}")
+    
+    for chunk_idx in range(0, total_docs, CHUNK_SIZE):
+        chunk_end = min(chunk_idx + CHUNK_SIZE, total_docs)
+        chunk_num = chunk_idx // CHUNK_SIZE + 1
         current_chunk = docs[chunk_idx:chunk_end]
         
-        # Process this chunk
-        texts_to_embed = []
-        doc_data = []
-        
-        for i, doc in enumerate(current_chunk):
-            text_content = doc.page_content
-            if len(text_content) > 65000:
-                text_content = text_content[:65000]
-                print(f"📄 Doc {chunk_idx + i + 1} truncated: {len(doc.page_content)} → {len(text_content)} chars")
-            
-            texts_to_embed.append(text_content)
-            doc_data.append({
-                "id": chunk_idx + i,
-                "text": text_content,
-                "metadata": doc.metadata if doc.metadata else {}
-            })
-        
-        # Generate embeddings with small batch size
-        print(f"🚀 Generating embeddings for {len(texts_to_embed)} documents...")
-        all_embeddings = process_embeddings_in_batches(texts_to_embed, batch_size=5)  # Very small batches
-        
-        # Prepare and insert data
-        data_to_insert = []
-        for doc_info, embedding in zip(doc_data, all_embeddings):
-            data_to_insert.append({
-                "id": doc_info["id"],
-                "vector": embedding,
-                "text": doc_info["text"],
-                "metadata": doc_info["metadata"]
-            })
-        
-        # Insert to Milvus
-        insert_result = milvus_client.insert(collection_name=collection_name, data=data_to_insert)
-        chunk_inserted = insert_result['insert_count']
+        # Process chunk
+        chunk_inserted = process_document_chunk(current_chunk, chunk_idx, chunk_num, total_chunks)
         total_inserted += chunk_inserted
         
-        print(f"✅ Chunk {chunk_num} complete: {chunk_inserted} docs inserted")
-        print(f"📈 Overall progress: {total_inserted}/{total_docs} ({(total_inserted/total_docs)*100:.1f}%)")
+        print(f"Chunk {chunk_num} complete: {chunk_inserted} docs inserted")
+        print(f"Progress: {total_inserted}/{total_docs} ({(total_inserted/total_docs)*100:.1f}%)")
         
-        # Critical: Free memory before next chunk
-        del texts_to_embed, doc_data, all_embeddings, data_to_insert, current_chunk
-        
-        # Brief pause between chunks
+        # Memory cleanup
+        del current_chunk
         if chunk_num < total_chunks:
-            print("⏱️ Memory cleanup pause (2s)...")
             time.sleep(2)
     
-    print(f"\n🎉 SUCCESS! All {total_inserted} documents processed and inserted!")
-    return docs
+    print(f"\nSuccessfully processed {total_inserted} documents!")
 
-def unstructured_document_loader():
-    """Load documents using UnstructuredLoader."""
-    # Collect file paths for PDF, DOCX, and HTML files
-    file_extensions = ["*.pdf", "*.docx", "*.html"]
-    file_paths = []
-
-    for ext in file_extensions:
-        file_paths.extend(Path(directory_path).glob(ext))
-
-    # Convert Path objects to strings
-    file_paths = [str(file) for file in file_paths]
-    
-    # Configure UnstructuredLoader with post-processors
-    loader = UnstructuredLoader(
-        file_paths, 
-        chunking_strategy="by_title", 
-        include_orig_elements=False,
-        post_processors=[
-            clean_extra_whitespace, 
-            replace_unicode_quotes, 
-            clean_dashes, 
-            group_broken_paragraphs
-        ]
-    )
-    
-    docs = loader.load()
-    print(f"Number of LangChain documents: {len(docs)}")
-    print(f"Length of first document: {len(docs[0].page_content)} characters")
-    print(f"First document preview: {docs[0].page_content[:200]}...")
-    
-    return docs
 
 def verify_insertion():
     """Verify that data was successfully inserted into Milvus."""
-    # Get collection statistics
-    stats = milvus_client.get_collection_stats(collection_name)
+    stats = milvus_client.get_collection_stats(COLLECTION_NAME)
     print(f"Collection stats: {stats}")
     
-    # Test search functionality with a sample query
-    test_query = "Questions by staff to other staff"
-    test_embedding = emb_text(test_query)
+    # Test search functionality
+    test_query = "Why should reasonable adjustments be made?"
+    test_embedding = generate_embedding(test_query)
     
     search_results = milvus_client.search(
-        collection_name=collection_name,
+        collection_name=COLLECTION_NAME,
         data=[test_embedding],
         limit=3,
         output_fields=["text", "metadata"]
@@ -252,17 +319,14 @@ def verify_insertion():
 if __name__ == "__main__":
     start_time = time.time()
     
-    print("="*60)
-    print("STARTING DOCUMENT PROCESSING AND MILVUS INSERTION")
-    print("="*60)
+    print("Starting document processing and Milvus insertion")
+    print("=" * 60)
     
     main()
     
-    print("\n" + "="*50)
-    print("VERIFYING DATA INSERTION")
-    print("="*50)
+    print("\nVerifying data insertion")
+    print("=" * 30)
     verify_insertion()
     
-    end_time = time.time()
-    elapsed_time = end_time - start_time
+    elapsed_time = time.time() - start_time
     print(f"\nTotal execution time: {elapsed_time:.2f} seconds")

prisma/migrations/20250103173917_init_data_layer/migration.sql

@@ -0,0 +1,159 @@
+-- CreateExtension
+CREATE EXTENSION IF NOT EXISTS "pgcrypto";
+
+-- CreateEnum
+CREATE TYPE "StepType" AS ENUM ('assistant_message', 'embedding', 'llm', 'retrieval', 'rerank', 'run', 'system_message', 'tool', 'undefined', 'user_message');
+
+-- CreateTable
+CREATE TABLE "Element" (
+    "id" TEXT NOT NULL DEFAULT gen_random_uuid(),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "threadId" TEXT,
+    "stepId" TEXT NOT NULL,
+    "metadata" JSONB NOT NULL,
+    "mime" TEXT,
+    "name" TEXT NOT NULL,
+    "objectKey" TEXT,
+    "url" TEXT,
+    "chainlitKey" TEXT,
+    "display" TEXT,
+    "size" TEXT,
+    "language" TEXT,
+    "page" INTEGER,
+    "props" JSONB,
+
+    CONSTRAINT "Element_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "User" (
+    "id" TEXT NOT NULL DEFAULT gen_random_uuid(),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "metadata" JSONB NOT NULL,
+    "identifier" TEXT NOT NULL,
+
+    CONSTRAINT "User_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Feedback" (
+    "id" TEXT NOT NULL DEFAULT gen_random_uuid(),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "stepId" TEXT,
+    "name" TEXT NOT NULL,
+    "value" DOUBLE PRECISION NOT NULL,
+    "comment" TEXT,
+
+    CONSTRAINT "Feedback_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Step" (
+    "id" TEXT NOT NULL DEFAULT gen_random_uuid(),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "parentId" TEXT,
+    "threadId" TEXT,
+    "input" TEXT,
+    "metadata" JSONB NOT NULL,
+    "name" TEXT,
+    "output" TEXT,
+    "type" "StepType" NOT NULL,
+    "showInput" TEXT DEFAULT 'json',
+    "isError" BOOLEAN DEFAULT false,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "endTime" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "Step_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "Thread" (
+    "id" TEXT NOT NULL DEFAULT gen_random_uuid(),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "deletedAt" TIMESTAMP(3),
+    "name" TEXT,
+    "metadata" JSONB NOT NULL,
+    "userId" TEXT,
+
+    CONSTRAINT "Thread_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "Element_stepId_idx" ON "Element"("stepId");
+
+-- CreateIndex
+CREATE INDEX "Element_threadId_idx" ON "Element"("threadId");
+
+-- CreateIndex
+CREATE INDEX "User_identifier_idx" ON "User"("identifier");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "User_identifier_key" ON "User"("identifier");
+
+-- CreateIndex
+CREATE INDEX "Feedback_createdAt_idx" ON "Feedback"("createdAt");
+
+-- CreateIndex
+CREATE INDEX "Feedback_name_idx" ON "Feedback"("name");
+
+-- CreateIndex
+CREATE INDEX "Feedback_stepId_idx" ON "Feedback"("stepId");
+
+-- CreateIndex
+CREATE INDEX "Feedback_value_idx" ON "Feedback"("value");
+
+-- CreateIndex
+CREATE INDEX "Feedback_name_value_idx" ON "Feedback"("name", "value");
+
+-- CreateIndex
+CREATE INDEX "Step_createdAt_idx" ON "Step"("createdAt");
+
+-- CreateIndex
+CREATE INDEX "Step_endTime_idx" ON "Step"("endTime");
+
+-- CreateIndex
+CREATE INDEX "Step_parentId_idx" ON "Step"("parentId");
+
+-- CreateIndex
+CREATE INDEX "Step_startTime_idx" ON "Step"("startTime");
+
+-- CreateIndex
+CREATE INDEX "Step_threadId_idx" ON "Step"("threadId");
+
+-- CreateIndex
+CREATE INDEX "Step_type_idx" ON "Step"("type");
+
+-- CreateIndex
+CREATE INDEX "Step_name_idx" ON "Step"("name");
+
+-- CreateIndex
+CREATE INDEX "Step_threadId_startTime_endTime_idx" ON "Step"("threadId", "startTime", "endTime");
+
+-- CreateIndex
+CREATE INDEX "Thread_createdAt_idx" ON "Thread"("createdAt");
+
+-- CreateIndex
+CREATE INDEX "Thread_name_idx" ON "Thread"("name");
+
+-- AddForeignKey
+ALTER TABLE "Element" ADD CONSTRAINT "Element_stepId_fkey" FOREIGN KEY ("stepId") REFERENCES "Step"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Element" ADD CONSTRAINT "Element_threadId_fkey" FOREIGN KEY ("threadId") REFERENCES "Thread"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Feedback" ADD CONSTRAINT "Feedback_stepId_fkey" FOREIGN KEY ("stepId") REFERENCES "Step"("id") ON DELETE SET NULL ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Step" ADD CONSTRAINT "Step_parentId_fkey" FOREIGN KEY ("parentId") REFERENCES "Step"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Step" ADD CONSTRAINT "Step_threadId_fkey" FOREIGN KEY ("threadId") REFERENCES "Thread"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "Thread" ADD CONSTRAINT "Thread_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE SET NULL ON UPDATE CASCADE;

prisma/migrations/20250108095538_add_tags_to_thread/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Thread" ADD COLUMN     "tags" TEXT[] DEFAULT ARRAY[]::TEXT[];

prisma/migrations/migration_lock.toml

@@ -0,0 +1,3 @@
+# Please do not edit this file manually
+# It should be added in your version-control system (e.g., Git)
+provider = "postgresql"

prisma/schema.prisma

@@ -0,0 +1,138 @@
+generator client {
+    provider             = "prisma-client-js"
+    interface            = "asyncio"
+    recursive_type_depth = 5
+    previewFeatures      = ["postgresqlExtensions"]
+}
+
+datasource db {
+    provider   = "postgresql"
+    url        = env("DATABASE_URL")
+    // Prisma migrations run through the direct URL. Replace as needed.
+    directUrl  = env("DATABASE_URL")
+    extensions = [pgcrypto]
+}
+
+model Element {
+    id        String   @id @default(dbgenerated("gen_random_uuid()"))
+    createdAt DateTime @default(now())
+    updatedAt DateTime @default(now()) @updatedAt
+
+    threadId  String?
+    stepId    String
+    metadata  Json
+    mime      String?
+    name      String
+    objectKey String?
+    url       String?
+    step      Step    @relation(fields: [stepId], references: [id], onDelete: Cascade)
+    thread    Thread? @relation(fields: [threadId], references: [id], onDelete: Cascade)
+
+    chainlitKey String?
+    display     String?
+    size        String?
+    language    String?
+    page        Int?
+    props       Json?
+
+    @@index([stepId])
+    @@index([threadId])
+}
+
+model User {
+    id         String   @id @default(dbgenerated("gen_random_uuid()"))
+    createdAt  DateTime @default(now())
+    updatedAt  DateTime @default(now()) @updatedAt
+    metadata   Json
+    identifier String
+    threads    Thread[]
+
+    @@unique([identifier])
+    @@index([identifier])
+}
+
+model Feedback {
+    id        String   @id @default(dbgenerated("gen_random_uuid()"))
+    createdAt DateTime @default(now())
+    updatedAt DateTime @default(now()) @updatedAt
+
+    stepId String?
+    Step   Step?   @relation(fields: [stepId], references: [id])
+
+    name  String
+    value Float
+
+    comment String?
+
+    @@index(createdAt)
+    @@index(name)
+    @@index(stepId)
+    @@index(value)
+    @@index([name, value])
+}
+
+model Step {
+    id        String   @id @default(dbgenerated("gen_random_uuid()"))
+    createdAt DateTime @default(now())
+    updatedAt DateTime @default(now()) @updatedAt
+    parentId  String?
+    threadId  String?
+
+    input     String?
+    metadata  Json
+    name      String?
+    output    String?
+    type      StepType
+    showInput String?  @default("json")
+    isError   Boolean? @default(false)
+
+    startTime DateTime
+    endTime   DateTime
+
+    elements Element[]
+    parent   Step?      @relation("ParentChild", fields: [parentId], references: [id], onDelete: Cascade)
+    children Step[]     @relation("ParentChild")
+    thread   Thread?    @relation(fields: [threadId], references: [id], onDelete: Cascade)
+    Feedback Feedback[]
+
+    @@index([createdAt])
+    @@index([endTime])
+    @@index([parentId])
+    @@index([startTime])
+    @@index([threadId])
+    @@index([type])
+    @@index([name])
+    @@index([threadId, startTime, endTime])
+}
+
+model Thread {
+    id        String    @id @default(dbgenerated("gen_random_uuid()"))
+    createdAt DateTime  @default(now())
+    updatedAt DateTime  @default(now()) @updatedAt
+    deletedAt DateTime?
+
+    name     String?
+    metadata Json
+    tags     String[] @default([])
+
+    elements Element[]
+    userId   String?
+    User     User?     @relation(fields: [userId], references: [id])
+    steps    Step[]
+
+    @@index([createdAt])
+    @@index([name])
+}
+
+enum StepType {
+    assistant_message
+    embedding
+    llm
+    retrieval
+    rerank
+    run
+    system_message
+    tool
+    undefined
+    user_message
+}

requirements.txt

@@ -13,3 +13,5 @@ fastapi>=0.100.0
 uvicorn>=0.20.0
 langchain_nebius>=0.1.0
 asyncpg>=0.29.0
+ragas>=0.1.0
+datasets>=2.0.0

test_chainlit.py

@@ -0,0 +1,24 @@
+import unittest
+import os
+from dotenv import load_dotenv
+from app import auth  # Import your Chainlit handler
+import chainlit as cl
+# Load environment variables
+load_dotenv()
+
+class TestClass(unittest.TestCase):
+    def test_authentication_valid_credentials(self):
+        # Test with valid credentials - should return a User object
+        user = auth("admin", os.getenv("PASSWORD"))
+        assert isinstance(user, cl.User)
+         
+    def test_authentication_invalid_credentials(self):
+        # Test with invalid credentials - should return None
+        user = auth("admin", "wrong_password")
+        assert not isinstance(user, cl.User)
+
+    def test_authentication_invalid_username(self):
+        # Test with invalid username - should return None
+        user = auth("wrong_user", os.getenv("PASSWORD"))
+        assert not isinstance(user, cl.User)
+        

test_ragas.ipynb

@@ -0,0 +1,447 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6bb3bb7d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "f:\\Dissertation\\prod-rag-chat\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:34:21 - Loaded .env file\n"
+     ]
+    }
+   ],
+   "source": [
+    "from datasets import Dataset\n",
+    "from app import retrieve_relevant_documents, emb_text, model, embedding_model\n",
+    "from langchain.chains.combine_documents import create_stuff_documents_chain\n",
+    "from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+    "from langchain.schema.runnable import RunnableLambda\n",
+    "from langchain_core.documents import Document"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e572fb31",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def setup_standalone_rag_chain():\n",
+    "    \"\"\"Setup a standalone RAG chain for testing without Chainlit session.\"\"\"\n",
+    "    \n",
+    "    def get_context_and_history(inputs):\n",
+    "        \"\"\"Retrieve context without session history.\"\"\"\n",
+    "        query = inputs[\"question\"]\n",
+    "        relevant_docs = retrieve_relevant_documents(query, limit=5)\n",
+    "        print(\"Relevant documents:\", relevant_docs[0] if relevant_docs else \"No documents found\")\n",
+    "        \n",
+    "        # Convert dictionaries to Document objects for LangChain\n",
+    "        doc_objects = []\n",
+    "        for doc in relevant_docs:\n",
+    "            doc_obj = Document(\n",
+    "                page_content=doc.get('text', ''),\n",
+    "                metadata=doc.get('metadata', {})\n",
+    "            )\n",
+    "            doc_objects.append(doc_obj)\n",
+    "\n",
+    "        return {\n",
+    "            \"question\": query,\n",
+    "            \"context\": doc_objects,\n",
+    "            \"history\": []  # Empty history for testing\n",
+    "        }\n",
+    "    \n",
+    "    system_prompt = \"\"\"You are a helpful assistant specialising in developing non-discriminatory competence standards and disability support, reasonable adjustments, and equality legislation.\n",
+    "\n",
+    "When answering questions, you should:\n",
+    "1. Use the provided context documents to inform your response\n",
+    "2. Be accurate and helpful\n",
+    "3. If the context doesn't contain relevant information, say so clearly\n",
+    "4. Always reply in English\n",
+    "5. Provide clear recommendations and examples wherever applicable\n",
+    "6. Do not make assumptions about the user's knowledge or background\n",
+    "7. If the user asks for a specific law or regulation, provide a brief explanation and cite relevant documents if available.\n",
+    "8. Do not overemphasize disability in your responses, but rather focus on the support and adjustments that can be made to ensure equality and inclusivity.\n",
+    "9. If the user query explicitly asks for a disability-related topic, provide a well-informed response based on the context documents.\n",
+    "\n",
+    "Context documents:\n",
+    "{context} \n",
+    "\n",
+    "Please provide a clear response using the above context\n",
+    "\"\"\"\n",
+    "\n",
+    "    prompt = ChatPromptTemplate.from_messages([\n",
+    "        (\"system\", system_prompt),\n",
+    "        MessagesPlaceholder(variable_name=\"history\"),\n",
+    "        (\"human\", \"{question}\"),\n",
+    "    ])\n",
+    "\n",
+    "    question_answer_chain = create_stuff_documents_chain(model, prompt)\n",
+    "    \n",
+    "    # Use a custom chain that properly handles our context and history\n",
+    "    def process_input_and_format(inputs):\n",
+    "        context_data = get_context_and_history(inputs)\n",
+    "        return {\n",
+    "            \"context\": context_data[\"context\"],\n",
+    "            \"question\": context_data[\"question\"],\n",
+    "            \"history\": context_data[\"history\"]\n",
+    "        }\n",
+    "    \n",
+    "    chain = RunnableLambda(process_input_and_format) | question_answer_chain\n",
+    "    \n",
+    "    return chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "330ee35d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# Setup the RAG chain\n",
+    "rag_chain = setup_standalone_rag_chain()\n",
+    "\n",
+    "questions = [\"What are Provisions, Criteria and Practices?\", \n",
+    "             \"What is 'reasonable'?\",\n",
+    "             \"What is 'substantial disadvantage'?\",\n",
+    "            ]\n",
+    "ground_truths = [\n",
+    "    \"\"\"The Equality and Human Rights Commission (EHRC) interprets PCPs as including:3 \n",
+    "+ arrangements (for example, for deciding who to admit)  \n",
+    "+ the way that education, or access to any benefit, service or facility is offered or provided  \n",
+    "+ one-off or discretionary decisions  \n",
+    "+ proposals or directions to do something in a particular way  \n",
+    "+ formal and informal policies  \n",
+    "+ rules\"\"\",\n",
+    "\n",
+    "    \"\"\"There are two key considerations of 'reasonableness' which can help when thinking through \n",
+    "when an adjustment may be reasonable:4  \n",
+    "+ Could the adjustment be practicable in its application (is it possible)? \n",
+    "+ Could the adjustment be effective in achieving its aim (will it work)? \n",
+    "There is no need to prove that the adjustment is practicable and effective in advance, just \n",
+    "that it might be. An adjustment should not be considered unreasonable if it does not remove \n",
+    "the disadvantage fully; an adjustment which partially removes or reduces substantial \n",
+    "disadvantage is also likely to be reasonable.\"\"\",\n",
+    "\n",
+    "    \"\"\"'Substantial' is defined in the Act as 'more than minor or trivial'. \n",
+    "Examples of disadvantage recognised by the EHRC include: \n",
+    "+ The additional time and effort expended by a disabled student \n",
+    "+ The inconvenience, indignity, discomfort, or perceived disadvantage suffered by a \n",
+    "disabled student \n",
+    "+ The loss of opportunity or diminished progress experienced by a disabled student. \"\"\"]\n",
+    "\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ba3810dd",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:34:53 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Relevant documents: {'text': 'What is a provision, criterion or practice? The phrase ‘provision, criterion or practice’ is not defined by the Act. These concepts should be construed widely so as to include, for example, any formal or informal policies, rules, practices, arrangements, criteria, procedures, activities or provisions. They can cover one-off decisions and actions. In simple terms, they are about the way an education provider does things. Example:', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 95, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': '3ae881ad6f81487213a9e234debf0921'}, 'score': 0.7780322432518005}\n",
+      "2025-08-16 16:34:53 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:02 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:03 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Relevant documents: {'text': '‘Reasonable’ means having regard to all of the circumstances including the nature of the act and how obviously discriminatory it is, the authority of the person making the statement and the knowledge that the helper has or ought to have.', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 36, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': 'c5e3a60e2a6ccc88e0eff961f645a962'}, 'score': 0.734176754951477}\n",
+      "2025-08-16 16:35:03 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:11 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:12 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "Relevant documents: {'text': 'The Act states that disadvantage must be substantial, which is defined as more than minor or trivial. Whether such a disadvantage exists in a particular case is a question of fact, and is assessed on an objective basis. s212(1)', 'metadata': {'source': 'data\\\\technical-guidance-further-higher-education.docx', 'file_directory': 'data', 'filename': 'technical-guidance-further-higher-education.docx', 'last_modified': '2025-07-02T21:00:50', 'page_number': 89, 'languages': ['eng'], 'filetype': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'category': 'CompositeElement', 'element_id': 'b9e8ef04daf9150c9f7e32736b53df5b'}, 'score': 0.8380770087242126}\n",
+      "2025-08-16 16:35:12 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:21 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "answers = []\n",
+    "contexts = []\n",
+    "\n",
+    "def clean_answer(answer):\n",
+    "    \"\"\"Remove <think></think> tags and content from the answer.\"\"\"\n",
+    "    import re\n",
+    "    # Remove everything between <think> and </think> tags, including the tags themselves\n",
+    "    cleaned = re.sub(r'<think>.*?</think>\\s*', '', answer, flags=re.DOTALL)\n",
+    "    return cleaned.strip()\n",
+    "\n",
+    "# Inference\n",
+    "for query in questions:\n",
+    "    # Get answer from the RAG chain\n",
+    "    answer = rag_chain.invoke({\"question\": query})\n",
+    "    # Clean the answer to remove thinking content\n",
+    "    cleaned_answer = clean_answer(answer)\n",
+    "    answers.append(cleaned_answer)\n",
+    "    \n",
+    "    # Get relevant documents for context\n",
+    "    relevant_docs = retrieve_relevant_documents(query, limit=5)\n",
+    "    context_texts = [doc['text'] for doc in relevant_docs]\n",
+    "    contexts.append(context_texts)\n",
+    "\n",
+    "# To dict\n",
+    "data = {\n",
+    "    \"question\": questions,\n",
+    "    \"answer\": answers,\n",
+    "    \"contexts\": contexts,\n",
+    "    \"reference\": ground_truths\n",
+    "}\n",
+    "\n",
+    "# Convert dict to dataset\n",
+    "dataset = Dataset.from_dict(data)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "3e016be2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:   0%|          | 0/12 [00:00<?, ?it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:22 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:26 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:26 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:27 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:27 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:28 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:   8%|▊         | 1/12 [00:06<01:13,  6.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:  17%|█▋        | 2/12 [00:07<00:32,  3.29s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:29 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:31 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:31 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:31 - HTTP Request: POST https://api.studio.nebius.ai/v1/embeddings \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:  25%|██▌       | 3/12 [00:09<00:23,  2.61s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:31 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:32 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:33 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:  33%|███▎      | 4/12 [00:11<00:19,  2.46s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:34 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating:  42%|████▏     | 5/12 [00:12<00:13,  1.88s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2025-08-16 16:35:35 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:35 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:35 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:36 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:38 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:39 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:39 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:41 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
+      "2025-08-16 16:35:43 - HTTP Request: POST https://api.studio.nebius.ai/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Evaluating: 100%|██████████| 12/12 [00:42<00:00,  3.51s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "from ragas import evaluate\n",
+    "from ragas.metrics import (\n",
+    "    faithfulness,\n",
+    "    answer_relevancy,\n",
+    "    context_recall,\n",
+    "    context_precision,\n",
+    ")\n",
+    "\n",
+    "result = evaluate(\n",
+    "    llm=model,\n",
+    "    embeddings=embedding_model,\n",
+    "    dataset = dataset, \n",
+    "    metrics=[\n",
+    "        context_precision,\n",
+    "        context_recall,\n",
+    "        faithfulness,\n",
+    "        answer_relevancy,\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "df = result.to_pandas()\n",
+    "\n",
+    "# evaluation_results = result.to_pandas()\n",
+    "\n",
+    "# display_columns = ['user_input', 'answer_relevancy', 'faithfulness', 'context_precision', 'context_recall']\n",
+    "# formatted_results = evaluation_results[display_columns].to_markdown(index=False, numalign=\"left\", stralign=\"left\")\n",
+    "\n",
+    "# print(formatted_results)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d8514ff3",
+   "metadata": {
+    "slideshow": {
+     "slide_type": "slide"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "                                     user_input  \\\n",
+      "0  What are Provisions, Criteria and Practices?   \n",
+      "1                         What is 'reasonable'?   \n",
+      "2           What is 'substantial disadvantage'?   \n",
+      "\n",
+      "                                  retrieved_contexts  \\\n",
+      "0  [What is a provision, criterion or practice? T...   \n",
+      "1  [‘Reasonable’ means having regard to all of th...   \n",
+      "2  [The Act states that disadvantage must be subs...   \n",
+      "\n",
+      "                                            response  \\\n",
+      "0  **Provisions, Criteria, and Practices (PCPs)**...   \n",
+      "1  The term **\"reasonable\"** in the context of di...   \n",
+      "2  **Substantial disadvantage** refers to a situa...   \n",
+      "\n",
+      "                                           reference  context_precision  \\\n",
+      "0  The Equality and Human Rights Commission (EHRC...           0.500000   \n",
+      "1  There are two key considerations of 'reasonabl...           0.500000   \n",
+      "2  'Substantial' is defined in the Act as 'more t...           0.866667   \n",
+      "\n",
+      "   context_recall  faithfulness  answer_relevancy  \n",
+      "0        0.857143      0.666667          0.759951  \n",
+      "1        0.500000      0.642857          0.616460  \n",
+      "2        0.500000      0.937500          0.767732  \n"
+     ]
+    }
+   ],
+   "source": [
+    "print(df)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

test_ragas.py

@@ -0,0 +1,165 @@
+from datasets import Dataset
+from app import retrieve_relevant_documents, emb_text, model, embedding_model
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.schema.runnable import RunnableLambda
+from langchain_core.documents import Document
+
+
+def setup_standalone_rag_chain():
+    """Setup a standalone RAG chain for testing without Chainlit session."""
+    
+    def get_context_and_history(inputs):
+        """Retrieve context without session history."""
+        query = inputs["question"]
+        relevant_docs = retrieve_relevant_documents(query, limit=5)
+        print("Relevant documents:", relevant_docs[0] if relevant_docs else "No documents found")
+        
+        # Convert dictionaries to Document objects for LangChain
+        doc_objects = []
+        for doc in relevant_docs:
+            doc_obj = Document(
+                page_content=doc.get('text', ''),
+                metadata=doc.get('metadata', {})
+            )
+            doc_objects.append(doc_obj)
+
+        return {
+            "question": query,
+            "context": doc_objects,
+            "history": []  # Empty history for testing
+        }
+    
+    system_prompt = """You are a helpful assistant specialising in developing non-discriminatory competence standards and disability support, reasonable adjustments, and equality legislation.
+
+When answering questions, you should:
+1. Use the provided context documents to inform your response
+2. Be accurate and helpful
+3. If the context doesn't contain relevant information, say so clearly
+4. Always reply in English
+5. Provide clear recommendations and examples wherever applicable
+6. Do not make assumptions about the user's knowledge or background
+7. If the user asks for a specific law or regulation, provide a brief explanation and cite relevant documents if available.
+8. Do not overemphasize disability in your responses, but rather focus on the support and adjustments that can be made to ensure equality and inclusivity.
+9. If the user query explicitly asks for a disability-related topic, provide a well-informed response based on the context documents.
+
+Context documents:
+{context} 
+
+Please provide a clear response using the above context
+"""
+
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", system_prompt),
+        MessagesPlaceholder(variable_name="history"),
+        ("human", "{question}"),
+    ])
+
+    question_answer_chain = create_stuff_documents_chain(model, prompt)
+    
+    # Use a custom chain that properly handles our context and history
+    def process_input_and_format(inputs):
+        context_data = get_context_and_history(inputs)
+        return {
+            "context": context_data["context"],
+            "question": context_data["question"],
+            "history": context_data["history"]
+        }
+    
+    chain = RunnableLambda(process_input_and_format) | question_answer_chain
+    
+    return chain
+
+# Setup the RAG chain
+rag_chain = setup_standalone_rag_chain()
+
+questions = ["What is a 'reasonable adjustment'?", 
+             "To whom do competence standards apply?",
+             "Do competence standards vary by subject?",
+            ]
+ground_truths = [
+    """The reasonable adjustments duty contains three requirements, which relate to changing
+how things are done, changing the built environment to avoid such a substantial
+disadvantage and providing auxiliary aids and services. Specifically:
+1. A duty to make reasonable adjustments to any provision, criterion or practice (PCP)
+which places disabled students at a substantial disadvantage
+2. A duty to make reasonable adjustments to physical features
+3. A duty to provide auxiliary aids (including services) """,
+
+    """The Act does not specify to whom competence standards may be applied but it is clear that 
+anti-discrimination provisions apply to prospective and current students (and in some cases 
+former students). 
+Providers commonly apply competence standards to:13 
++ Applicants, to determine whether they have the knowledge and skills necessary to 
+participate in and complete a course of study 
++ Students, to determine whether they are ready to progress to the next year/stage of
+study, and to determine whether they have demonstrated the requirements in order to be
+awarded a qualification that necessitates a competence standard to be applied.""",
+
+    """Competence standards can - and should - vary between courses of study.  What may 
+constitute a competence standard in one subject area may not be justifiable in another. """]
+
+
+answers = []
+contexts = []
+
+def clean_answer(answer):
+    """Remove <think></think> tags and content from the answer."""
+    import re
+    # Remove everything between <think> and </think> tags, including the tags themselves
+    cleaned = re.sub(r'<think>.*?</think>\s*', '', answer, flags=re.DOTALL)
+    return cleaned.strip()
+
+# Inference
+for query in questions:
+    # Get answer from the RAG chain
+    answer = rag_chain.invoke({"question": query})
+    # Clean the answer to remove thinking content
+    cleaned_answer = clean_answer(answer)
+    answers.append(cleaned_answer)
+    
+    # Get relevant documents for context
+    relevant_docs = retrieve_relevant_documents(query, limit=5)
+    context_texts = [doc['text'] for doc in relevant_docs]
+    contexts.append(context_texts)
+
+# To dict
+data = {
+    "question": questions,
+    "answer": answers,
+    "contexts": contexts,
+    "reference": ground_truths
+}
+
+# Convert dict to dataset
+dataset = Dataset.from_dict(data)
+
+
+from ragas import evaluate
+from ragas.metrics import (
+    faithfulness,
+    answer_relevancy,
+    context_recall,
+    context_precision,
+)
+
+result = evaluate(
+    llm=model,
+    embeddings=embedding_model,
+    dataset = dataset, 
+    metrics=[
+        context_precision,
+        context_recall,
+        faithfulness,
+        answer_relevancy,
+    ],
+)
+
+# df = result.to_pandas()
+
+evaluation_results = result.to_pandas()
+
+display_columns = ['user_input', 'answer_relevancy', 'faithfulness', 'context_precision', 'context_recall']
+formatted_results = evaluation_results[display_columns].to_markdown(index=False, numalign="left", stralign="left")
+
+print(formatted_results)

test_vector_search.py

@@ -0,0 +1,62 @@
+import os
+import time
+from langchain_nebius import NebiusEmbeddings
+from pydantic import SecretStr
+from pymilvus import MilvusClient
+# Configuration constants
+MILVUS_URI = os.getenv("MILVUS_URI", "http://localhost:19530")
+COLLECTION_NAME = "my_rag_collection"
+DOCUMENT_DIR = "data/"
+EMBEDDING_DIMENSION = 4096
+
+milvus_client = MilvusClient(uri=MILVUS_URI)
+
+TEXT_MAX_LENGTH = 65000
+CHUNK_SIZE = 100
+BATCH_SIZE = 5
+
+
+embedding_model = NebiusEmbeddings(
+    api_key=SecretStr(os.getenv("NEBIUS_API_KEY", os.getenv("OPENAI_API_KEY"))),
+    model="Qwen/Qwen3-Embedding-8B",
+    base_url="https://api.studio.nebius.ai/v1"
+)
+
+def generate_embedding(text):
+    """Generate embedding for a single text."""
+    return embedding_model.embed_query(text)
+
+def verify_insertion():
+    """Verify that data was successfully inserted into Milvus."""
+    stats = milvus_client.get_collection_stats(COLLECTION_NAME)
+    print(f"Collection stats: {stats}")
+    
+    # Test search functionality
+    test_query = "What are competence standards and their purpose?"
+    test_embedding = generate_embedding(test_query)
+    
+    search_results = milvus_client.search(
+        collection_name=COLLECTION_NAME,
+        data=[test_embedding],
+        limit=3,
+        output_fields=["text", "metadata"]
+    )
+    
+    print(f"\nTest search results for '{test_query}':")
+    for i, result in enumerate(search_results[0]):
+        print(f"Result {i+1}:")
+        print(f"  Score: {result['distance']:.4f}")
+        print(f"  Text preview: {result['entity']['text'][:300]}...")
+        print(f"  Metadata: {result['entity']['metadata']}")
+        print("-" * 50)
+
+if __name__ == "__main__":
+    start_time = time.time()
+    print("=" * 60)
+   
+    print("\n Starting test search")
+    print("=" * 30)
+    verify_insertion()
+    
+    elapsed_time = time.time() - start_time
+    print(f"\nTotal execution time: {elapsed_time:.2f} seconds")