Update app.py

app.py

@@ -1,101 +1,226 @@
-# Force install sentencepiece
-import sys
-import subprocess
-
-def install_package(package):
-    subprocess.check_call([sys.executable, "-m", "pip", "install", package])
-
-try:
-    import sentencepiece
-    print("SentencePiece is already installed")
-except ImportError:
-    print("Installing SentencePiece...")
-    install_package("sentencepiece==0.1.99")
-    print("SentencePiece installed successfully")
-
-# Import other required libraries
-import gradio as gr
 import os
 import re
 import torch
+import gradio as gr
 import numpy as np
 from pathlib import Path
+from tqdm import tqdm
+import json
+
+# PDF processing
 import PyPDF2
-from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSeq2SeqLM
+
+# LLM and embeddings
+from transformers import AutoTokenizer, AutoModelForCausalLM
 from sentence_transformers import SentenceTransformer
+
+# RAG components
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import FAISS
 from langchain.schema import Document
 from langchain.embeddings import HuggingFaceEmbeddings
-import spaces
 
-# Global variables to store model state
-model = None
-tokenizer = None
-assistant = None
-model_type = "primary"  # Track if we're using primary or fallback model
+# Arabic text processing
+import arabic_reshaper
+from bidi.algorithm import get_display
 
-# Create the Vision 2030 Assistant class
-class Vision2030Assistant:
-    def __init__(self, model, tokenizer, vector_store, model_type="primary"):
-        self.model = model
-        self.tokenizer = tokenizer
-        self.vector_store = vector_store
-        self.model_type = model_type
-        self.conversation_history = []
+# Evaluation
+from rouge_score import rouge_scorer
+
+# Helper functions from your notebook
+def detect_language(text):
+    """Detect if text is primarily Arabic or English"""
+    # Simple heuristic: count Arabic characters
+    arabic_chars = re.findall(r'[\u0600-\u06FF]', text)
+    is_arabic = len(arabic_chars) > len(text) * 0.5
+    return "arabic" if is_arabic else "english"
+
+def safe_tokenize(text):
+    """Pure regex tokenizer with no NLTK dependency"""
+    if not text:
+        return []
+    # Replace punctuation with spaces around them
+    text = re.sub(r'([.,!?;:()\[\]{}"\'/\\])', r' \1 ', text)
+    # Split on whitespace and filter empty strings
+    return [token for token in re.split(r'\s+', text.lower()) if token]
+
+# Evaluation metric functions
+def calculate_bleu(prediction, reference):
+    """Calculate BLEU score without any NLTK dependency"""
+    # Tokenize texts using our own tokenizer
+    pred_tokens = safe_tokenize(prediction.lower())
+    ref_tokens = [safe_tokenize(reference.lower())]
+    
+    # If either is empty, return 0
+    if not pred_tokens or not ref_tokens[0]:
+        return {"bleu_1": 0, "bleu_2": 0, "bleu_4": 0}
+    
+    # Get n-grams function
+    def get_ngrams(tokens, n):
+        return [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]
+    
+    # Calculate precision for each n-gram level
+    precisions = []
+    for n in range(1, 5):  # 1-gram to 4-gram
+        if len(pred_tokens) < n:
+            precisions.append(0)
+            continue
+            
+        pred_ngrams = get_ngrams(pred_tokens, n)
+        ref_ngrams = get_ngrams(ref_tokens[0], n)
         
-    def answer(self, user_query):
-        # Detect language
-        language = detect_language(user_query)
+        # Count matches
+        matches = sum(1 for ng in pred_ngrams if ng in ref_ngrams)
         
-        # Add user query to conversation history
-        self.conversation_history.append({"role": "user", "content": user_query})
+        # Calculate precision
+        if pred_ngrams:
+            precisions.append(matches / len(pred_ngrams))
+        else:
+            precisions.append(0)
+    
+    # Return BLEU scores
+    return {
+        "bleu_1": precisions[0],
+        "bleu_2": (precisions[0] * precisions[1]) ** 0.5 if len(precisions) > 1 else 0,
+        "bleu_4": (precisions[0] * precisions[1] * precisions[2] * precisions[3]) ** 0.25 if len(precisions) > 3 else 0
+    }
+
+def calculate_meteor(prediction, reference):
+    """Simple word overlap metric as METEOR alternative"""
+    # Tokenize with our custom tokenizer
+    pred_tokens = set(safe_tokenize(prediction.lower()))
+    ref_tokens = set(safe_tokenize(reference.lower()))
+    
+    # Calculate Jaccard similarity as METEOR alternative
+    if not pred_tokens or not ref_tokens:
+        return 0
         
-        # Get the full conversation context
-        conversation_context = "\n".join([
-            f"{'User' if msg['role'] == 'user' else 'Assistant'}: {msg['content']}"
-            for msg in self.conversation_history[-6:]  # Keep last 3 turns (6 messages)
+    intersection = len(pred_tokens.intersection(ref_tokens))
+    union = len(pred_tokens.union(ref_tokens))
+    
+    return intersection / union if union > 0 else 0
+
+def calculate_f1_precision_recall(prediction, reference):
+    """Calculate word-level F1, precision, and recall with custom tokenizer"""
+    # Tokenize with our custom tokenizer
+    pred_tokens = set(safe_tokenize(prediction.lower()))
+    ref_tokens = set(safe_tokenize(reference.lower()))
+    
+    # Calculate overlap
+    common = pred_tokens.intersection(ref_tokens)
+    
+    # Calculate precision, recall, F1
+    precision = len(common) / len(pred_tokens) if pred_tokens else 0
+    recall = len(common) / len(ref_tokens) if ref_tokens else 0
+    f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0
+    
+    return {'precision': precision, 'recall': recall, 'f1': f1}
+
+# Load PDFs and create vector store
+def process_pdfs(pdf_files):
+    """Process uploaded PDF documents and return document objects"""
+    documents = []
+    
+    for pdf_path in pdf_files:
+        try:
+            text = ""
+            with open(pdf_path, 'rb') as file:
+                reader = PyPDF2.PdfReader(file)
+                for page in reader.pages:
+                    page_text = page.extract_text()
+                    if page_text:  # If we got text from this page
+                        text += page_text + "\n\n"
+            
+            if text.strip():  # If we got some text
+                doc = Document(
+                    page_content=text,
+                    metadata={"source": pdf_path, "filename": os.path.basename(pdf_path)}
+                )
+                documents.append(doc)
+                print(f"Successfully processed: {pdf_path}")
+            else:
+                print(f"Warning: No text extracted from {pdf_path}")
+        except Exception as e:
+            print(f"Error processing {pdf_path}: {e}")
+    
+    print(f"Processed {len(documents)} PDF documents")
+    return documents
+
+def create_vector_store(documents):
+    """Split documents into chunks and create a FAISS vector store"""
+    # Text splitter for breaking documents into chunks
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=50,
+        separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""]
+    )
+    
+    # Split documents into chunks
+    chunks = []
+    for doc in documents:
+        doc_chunks = text_splitter.split_text(doc.page_content)
+        # Preserve metadata for each chunk
+        chunks.extend([
+            Document(page_content=chunk, metadata=doc.metadata)
+            for chunk in doc_chunks
         ])
+    
+    print(f"Created {len(chunks)} chunks from {len(documents)} documents")
+    
+    # Create a proper embedding function for LangChain
+    embedding_function = HuggingFaceEmbeddings(
+        model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
+    )
+    
+    # Create FAISS index
+    vector_store = FAISS.from_documents(
+        chunks,
+        embedding_function
+    )
+    
+    return vector_store
+
+def load_model_and_tokenizer():
+    """Load the ALLaM-7B model and tokenizer with error handling"""
+    model_name = "ALLaM-AI/ALLaM-7B-Instruct-preview"
+    print(f"Loading model: {model_name}")
+
+    try:
+        # First attempt with AutoTokenizer
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_name,
+            trust_remote_code=True,
+            use_fast=False
+        )
         
-        # Enhance query with conversation context for better retrieval
-        enhanced_query = f"{conversation_context}\n{user_query}"
-        
-        # Retrieve relevant contexts
-        contexts = retrieve_context(enhanced_query, self.vector_store, top_k=5)
+        # Load model with appropriate settings for ALLaM
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype=torch.bfloat16,  # Use bfloat16 for better compatibility
+            trust_remote_code=True,
+            device_map="auto",
+        )
         
-        # Generate response based on model type
-        if self.model_type == "primary":
-            response = generate_response_primary(user_query, contexts, self.model, self.tokenizer, language)
-        else:
-            response = generate_response_fallback(user_query, contexts, self.model, self.tokenizer, language)
+        print("Model loaded successfully with AutoTokenizer!")
         
-        # Add response to conversation history
-        self.conversation_history.append({"role": "assistant", "content": response})
+    except Exception as e:
+        print(f"First loading attempt failed: {e}")
+        print("Trying alternative loading approach...")
         
-        # Also return sources for transparency
-        sources = [ctx.get("source", "Unknown") for ctx in contexts]
-        unique_sources = list(set(sources))
+        # Try with specific tokenizer class if the first attempt fails
+        from transformers import LlamaTokenizer
         
-        # Format the response with sources
-        if unique_sources:
-            source_text = "\n\nSources: " + ", ".join([os.path.basename(src) for src in unique_sources])
-            response_with_sources = response + source_text
-        else:
-            response_with_sources = response
+        tokenizer = LlamaTokenizer.from_pretrained(model_name)
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype=torch.float16,
+            trust_remote_code=True,
+            device_map="auto",
+        )
         
-        return response_with_sources
+        print("Model loaded successfully with LlamaTokenizer!")
     
-    def reset_conversation(self):
-        """Reset the conversation history"""
-        self.conversation_history = []
-        return "Conversation has been reset."
-
-# Helper functions
-def detect_language(text):
-    """Detect if text is primarily Arabic or English"""
-    arabic_chars = re.findall(r'[\u0600-\u06FF]', text)
-    is_arabic = len(arabic_chars) > len(text) * 0.5
-    return "arabic" if is_arabic else "english"
+    return model, tokenizer
 
 def retrieve_context(query, vector_store, top_k=5):
     """Retrieve most relevant document chunks for a given query"""
@@ -113,9 +238,8 @@ def retrieve_context(query, vector_store, top_k=5):
     
     return contexts
 
-@spaces.GPU
-def generate_response_primary(query, contexts, model, tokenizer, language="auto"):
-    """Generate a response using ALLaM model"""
+def generate_response(query, contexts, model, tokenizer, language="auto"):
+    """Generate a response using retrieved contexts with ALLaM-specific formatting"""
     # Auto-detect language if not specified
     if language == "auto":
         language = detect_language(query)
@@ -175,403 +299,332 @@ Question: {query} [/INST]</s>"""
         # Fallback response
         return "I apologize, but I encountered an error while generating a response."
 
-@spaces.GPU
-def generate_response_fallback(query, contexts, model, tokenizer, language="auto"):
-    """Generate a response using the fallback model (BLOOM or mBART)"""
-    # Auto-detect language if not specified
-    if language == "auto":
-        language = detect_language(query)
-    
-    # Format the prompt based on language
-    if language == "arabic":
-        system_prompt = (
-            "أنت مساعد افتراضي يهتم برؤية السعودية 2030. استخدم السياق التالي للإجابة على السؤال: "
-        )
-    else:
-        system_prompt = (
-            "You are a virtual assistant for Saudi Vision 2030. Use the following context to answer the question: "
-        )
-    
-    # Combine retrieved contexts
-    context_text = "\n\n".join([f"Document: {ctx['content']}" for ctx in contexts])
-    
-    # Format prompt for fallback model (simpler format)
-    prompt = f"{system_prompt}\n\nContext:\n{context_text}\n\nQuestion: {query}\n\nAnswer:"
-    
-    try:
-        # Generate with fallback model
-        inputs = tokenizer(prompt, return_tensors="pt", max_length=1024, truncation=True).to(model.device)
+# Assistant class
+class Vision2030Assistant:
+    def __init__(self, model, tokenizer, vector_store):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.vector_store = vector_store
+        self.conversation_history = []
         
-        outputs = model.generate(
-            inputs.input_ids,
-            attention_mask=inputs.attention_mask,
-            max_length=inputs.input_ids.shape[1] + 512,
-            temperature=0.7,
-            top_p=0.9,
-            do_sample=True,
-            pad_token_id=tokenizer.eos_token_id
-        )
+    def answer(self, user_query):
+        """Process a user query and return a response with sources"""
+        # Detect language
+        language = detect_language(user_query)
         
-        # For most models, this is how we extract the response
-        response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+        # Add user query to conversation history
+        self.conversation_history.append({"role": "user", "content": user_query})
         
-        # Cleanup and return
-        return response.strip()
-    
-    except Exception as e:
-        print(f"Error during fallback generation: {e}")
-        return "I apologize, but I encountered an error while generating a response with the fallback model."
-
-def process_pdf_files(pdf_files):
-    """Process PDF files and create documents"""
-    documents = []
-    
-    for pdf_file in pdf_files:
-        try:
-            # Save the uploaded file temporarily
-            temp_path = f"temp_{pdf_file.name}"
-            with open(temp_path, "wb") as f:
-                f.write(pdf_file.read())
-            
-            # Extract text
-            text = ""
-            with open(temp_path, 'rb') as file:
-                reader = PyPDF2.PdfReader(file)
-                for page in reader.pages:
-                    page_text = page.extract_text()
-                    if page_text:
-                        text += page_text + "\n\n"
-            
-            # Clean up
-            os.remove(temp_path)
-            
-            if text.strip():  # If we got some text
-                doc = Document(
-                    page_content=text,
-                    metadata={"source": pdf_file.name, "filename": pdf_file.name}
-                )
-                documents.append(doc)
-                print(f"Successfully processed: {pdf_file.name}")
-            else:
-                print(f"Warning: No text extracted from {pdf_file.name}")
-        except Exception as e:
-            print(f"Error processing {pdf_file.name}: {e}")
-    
-    print(f"Processed {len(documents)} PDF documents")
-    return documents
-
-def create_vector_store(documents):
-    """Create a vector store from documents"""
-    # Text splitter for breaking documents into chunks
-    text_splitter = RecursiveCharacterTextSplitter(
-        chunk_size=500,
-        chunk_overlap=50,
-        separators=["\n\n", "\n", ".", "!", "?", ",", " ", ""]
-    )
-    
-    # Split documents into chunks
-    chunks = []
-    for doc in documents:
-        doc_chunks = text_splitter.split_text(doc.page_content)
-        # Preserve metadata for each chunk
-        chunks.extend([
-            Document(page_content=chunk, metadata=doc.metadata)
-            for chunk in doc_chunks
+        # Get the full conversation context
+        conversation_context = "\n".join([
+            f"{'User' if msg['role'] == 'user' else 'Assistant'}: {msg['content']}"
+            for msg in self.conversation_history[-6:]  # Keep last 3 turns (6 messages)
         ])
-    
-    print(f"Created {len(chunks)} chunks from {len(documents)} documents")
-    
-    # Create embedding function
-    embedding_function = HuggingFaceEmbeddings(
-        model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
-    )
-    
-    # Create FAISS index
-    vector_store = FAISS.from_documents(chunks, embedding_function)
-    return vector_store
-
-# Attempt to create mock documents if none are available yet
-def create_mock_documents():
-    """Create mock documents about Vision 2030"""
-    documents = []
-    
-    # Sample content about Vision 2030 in both languages
-    samples = [
-        {
-            "content": "رؤية السعودية 2030 هي خطة استراتيجية تهدف إلى تنويع الاقتصاد السعودي وتقليل الاعتماد على النفط مع تطوير قطاعات مختلفة مثل الصحة والتعليم والسياحة.",
-            "source": "vision2030_overview_ar.txt"
-        },
-        {
-            "content": "Saudi Vision 2030 is a strategic framework aiming to diversify Saudi Arabia's economy and reduce dependence on oil, while developing sectors like health, education, and tourism.",
-            "source": "vision2030_overview_en.txt"
-        },
-        {
-            "content": "تشمل الأهداف الاقتصادية لرؤية 2030 زيادة مساهمة القطاع الخاص من 40% إلى 65% من الناتج المحلي الإجمالي، ورفع نسبة الصادرات غير النفطية من 16% إلى 50% من الناتج المحلي الإجمالي غير النفطي، وخفض البطالة إلى 7%.",
-            "source": "economic_goals_ar.txt"
-        },
-        {
-            "content": "The economic goals of Vision 2030 include increasing private sector contribution from 40% to 65% of GDP, raising non-oil exports from 16% to 50%, and reducing unemployment from 11.6% to 7%.",
-            "source": "economic_goals_en.txt"
-        },
-        {
-            "content": "تركز رؤية 2030 على زيادة مشاركة المرأة في سوق العمل من 22% إلى 30% بحلول عام 2030، مع توفير فرص متساوية في التعليم والعمل.",
-            "source": "women_empowerment_ar.txt"
-        },
-        {
-            "content": "Vision 2030 emphasizes increasing women's participation in the workforce from 22% to 30% by 2030, while providing equal opportunities in education and employment.",
-            "source": "women_empowerment_en.txt"
-        }
-    ]
-    
-    # Create documents from samples
-    for sample in samples:
-        doc = Document(
-            page_content=sample["content"],
-            metadata={"source": sample["source"], "filename": sample["source"]}
-        )
-        documents.append(doc)
-    
-    print(f"Created {len(documents)} mock documents")
-    return documents
-
-@spaces.GPU
-def load_primary_model():
-    """Load the ALLaM-7B model with error handling"""
-    global model, tokenizer, model_type
-    
-    if model is not None and tokenizer is not None and model_type == "primary":
-        return "Primary model (ALLaM-7B) already loaded"
-    
-    model_name = "ALLaM-AI/ALLaM-7B-Instruct-preview"
-    print(f"Loading primary model: {model_name}")
-
-    try:
-        # Try to import sentencepiece explicitly first
-        import sentencepiece as spm
-        print("SentencePiece imported successfully")
         
-        # First attempt with AutoTokenizer and explicit trust_remote_code
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_name,
-            trust_remote_code=True,
-            use_fast=False
-        )
+        # Enhance query with conversation context for better retrieval
+        enhanced_query = f"{conversation_context}\n{user_query}"
         
-        # Load model with appropriate settings for ALLaM
-        model = AutoModelForCausalLM.from_pretrained(
-            model_name,
-            torch_dtype=torch.bfloat16,
-            trust_remote_code=True,
-            device_map="auto",
-        )
+        # Retrieve relevant contexts
+        contexts = retrieve_context(enhanced_query, self.vector_store, top_k=5)
         
-        model_type = "primary"
-        return "Primary model (ALLaM-7B) loaded successfully!"
+        # Generate response
+        response = generate_response(user_query, contexts, self.model, self.tokenizer, language)
         
-    except Exception as e:
-        error_msg = f"Primary model loading failed: {e}"
-        print(error_msg)
-        return error_msg
-
-@spaces.GPU
-def load_fallback_model():
-    """Load the fallback model (BLOOM-7B1) when ALLaM fails"""
-    global model, tokenizer, model_type
-    
-    if model is not None and tokenizer is not None and model_type == "fallback":
-        return "Fallback model already loaded"
-    
-    try:
-        print("Loading fallback model: BLOOM-7B1...")
+        # Add response to conversation history
+        self.conversation_history.append({"role": "assistant", "content": response})
         
-        # Use BLOOM model as fallback (it doesn't need SentencePiece)
-        tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-7b1")
-        model = AutoModelForCausalLM.from_pretrained(
-            "bigscience/bloom-7b1",
-            torch_dtype=torch.bfloat16,
-            device_map="auto",
-            load_in_8bit=True  # Reduce memory usage
-        )
+        # Also return sources for transparency
+        sources = [ctx.get("source", "Unknown") for ctx in contexts]
+        unique_sources = list(set(sources))
         
-        model_type = "fallback"
-        return "Fallback model (BLOOM-7B1) loaded successfully!"
-    except Exception as e:
-        return f"Fallback model loading failed: {e}"
-
-def load_mbart_model():
-    """Load mBART as a second fallback option"""
-    global model, tokenizer, model_type
+        return response, unique_sources, contexts
     
-    try:
-        print("Loading mBART multilingual model...")
-        
-        model_name = "facebook/mbart-large-50-many-to-many-mmt"
-        tokenizer = AutoTokenizer.from_pretrained(model_name)
-        model = AutoModelForSeq2SeqLM.from_pretrained(
-            model_name,
-            torch_dtype=torch.float16,
-            device_map="auto",
-            load_in_8bit=True
-        )
-        
-        model_type = "mbart"
-        return "mBART multilingual model loaded successfully!"
-    except Exception as e:
-        return f"mBART model loading failed: {e}"
+    def reset_conversation(self):
+        """Reset the conversation history"""
+        self.conversation_history = []
+        return "Conversation has been reset."
 
-# Gradio Interface Functions
-def process_pdfs(pdf_files):
-    if not pdf_files:
-        return "No files uploaded. Please upload PDF documents about Vision 2030."
-    
-    documents = process_pdf_files(pdf_files)
-    
-    if not documents:
-        return "Failed to extract text from the uploaded PDFs."
+# Sample evaluation data (subset)
+sample_evaluation_data = [
+    {
+        "query": "ما هي رؤية السعودية 2030؟",
+        "reference": "رؤية السعودية 2030 هي خطة استراتيجية تهدف إلى تنويع الاقتصاد السعودي وتقليل الاعتماد على النفط مع تطوير قطاعات مختلفة مثل الصحة والتعليم والسياحة.",
+        "category": "overview",
+        "language": "arabic"
+    },
+    {
+        "query": "What is Saudi Vision 2030?",
+        "reference": "Saudi Vision 2030 is a strategic framework aiming to diversify Saudi Arabia's economy and reduce dependence on oil, while developing sectors like health, education, and tourism.",
+        "category": "overview",
+        "language": "english"
+    },
+    {
+        "query": "ما هي الأهداف الاقتصادية لرؤية 2030؟",
+        "reference": "تشمل الأهداف الاقتصادية زيادة مساهمة القطاع الخاص إلى 65%، وزيادة الصادرات غير النفطية إلى 50% من الناتج المحلي غير النفطي، وخفض البطالة إلى 7%.",
+        "category": "economic",
+        "language": "arabic"
+    },
+    {
+        "query": "What are the economic goals of Vision 2030?",
+        "reference": "The economic goals of Vision 2030 include increasing private sector contribution from 40% to 65% of GDP, raising non-oil exports from 16% to 50%, reducing unemployment from 11.6% to 7%.",
+        "category": "economic",
+        "language": "english"
+    },
+    {
+        "query": "How does Vision 2030 support small and medium enterprises (SMEs)?",
+        "reference": "Vision 2030 supports SMEs by increasing their GDP contribution, facilitating access to funding, and reducing regulatory obstacles.",
+        "category": "economic",
+        "language": "english"
+    }
+]
+
+# Global variables for storing state
+ASSISTANT = None
+MODEL = None
+TOKENIZER = None
+VECTOR_STORE = None
+PDF_PATHS = ["vision2030_docs/saudi_vision203.pdf", "vision2030_docs/saudi_vision2030_ar.pdf"]
+
+# Initialize evaluation
+rouge_scorer_instance = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], use_stemmer=True)
+
+def initialize_system():
+    global MODEL, TOKENIZER, VECTOR_STORE, ASSISTANT
     
-    global assistant, model, tokenizer
+    # Try to load from saved files first
+    if os.path.exists("data/vision2030_vector_store"):
+        print("Loading vector store from saved file...")
+        try:
+            embedding_function = HuggingFaceEmbeddings(
+                model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
+            )
+            VECTOR_STORE = FAISS.load_local("data/vision2030_vector_store", embedding_function)
+            print("Vector store loaded successfully!")
+        except Exception as e:
+            print(f"Error loading vector store: {e}")
+            VECTOR_STORE = None
     
-    # Ensure model is loaded
-    if model is None or tokenizer is None:
-        return "Please load a model first (primary or fallback) before processing documents."
+    # If vector store not loaded, process PDFs and create it
+    if VECTOR_STORE is None:
+        print("Processing PDF documents...")
+        vision2030_docs = process_pdfs(PDF_PATHS)
+        
+        if not vision2030_docs:
+            return "Error: No documents were processed. Cannot continue."
+        
+        print("Creating vector store...")
+        VECTOR_STORE = create_vector_store(vision2030_docs)
+        
+        # Save the vector store for future use
+        os.makedirs("data", exist_ok=True)
+        VECTOR_STORE.save_local("data/vision2030_vector_store")
+        print("Vector store saved to data/vision2030_vector_store")
     
-    # Create vector store
-    vector_store = create_vector_store(documents)
+    # Load model and tokenizer
+    print("Loading ALLaM-7B model...")
+    MODEL, TOKENIZER = load_model_and_tokenizer()
     
     # Initialize assistant
-    assistant = Vision2030Assistant(model, tokenizer, vector_store, model_type)
+    ASSISTANT = Vision2030Assistant(MODEL, TOKENIZER, VECTOR_STORE)
+    print("Vision 2030 Assistant initialized successfully!")
     
-    return f"Successfully processed {len(documents)} documents. The assistant is ready to use!"
+    return "System initialized and ready!"
 
-def use_mock_documents():
-    """Use mock documents when no PDFs are available"""
-    documents = create_mock_documents()
-    
-    global assistant, model, tokenizer
-    
-    # Ensure model is loaded
-    if model is None or tokenizer is None:
-        return "Please load a model first (primary or fallback) before using mock documents."
+def process_query(query, reference=None):
+    """Process a user query and return the response with evaluation if reference is provided"""
+    if ASSISTANT is None:
+        return "System not initialized. Please initialize first.", "", "", "", ""
+    
+    # Process query
+    response, sources, contexts = ASSISTANT.answer(query)
+    
+    # Additional details
+    language = detect_language(query)
+    source_text = "\n".join([f"Source: {s}" for s in sources])
+    context_text = "\n\n".join([f"Context {i+1}: {ctx['content'][:200]}..." for i, ctx in enumerate(contexts)])
+    
+    # Calculate metrics if reference is provided
+    metrics_text = ""
+    if reference:
+        # ROUGE scores
+        rouge_scores = rouge_scorer_instance.score(response, reference)
+        
+        # BLEU scores
+        bleu_scores = calculate_bleu(response, reference)
+        
+        # METEOR score
+        meteor = calculate_meteor(response, reference)
+        
+        # F1, Precision, Recall
+        word_metrics = calculate_f1_precision_recall(response, reference)
+        
+        # Format metrics text
+        metrics_text = f"""
+## Evaluation Metrics:
+- **ROUGE-1**: {rouge_scores['rouge1'].fmeasure:.4f}
+- **ROUGE-L**: {rouge_scores['rougeL'].fmeasure:.4f}
+- **BLEU-1**: {bleu_scores['bleu_1']:.4f}
+- **BLEU-4**: {bleu_scores['bleu_4']:.4f}
+- **METEOR**: {meteor:.4f}
+- **Word F1**: {word_metrics['f1']:.4f}
+- **Word Precision**: {word_metrics['precision']:.4f}
+- **Word Recall**: {word_metrics['recall']:.4f}
+        """
+    
+    return response, source_text, context_text, metrics_text, language
+
+def evaluate_sample(sample_index):
+    """Evaluate a sample from the predefined evaluation dataset"""
+    if sample_index < 0 or sample_index >= len(sample_evaluation_data):
+        return "Invalid sample index", "", "", "", ""
     
-    # Create vector store
-    vector_store = create_vector_store(documents)
+    sample = sample_evaluation_data[sample_index]
+    query = sample["query"]
+    reference = sample["reference"]
     
-    # Initialize assistant
-    assistant = Vision2030Assistant(model, tokenizer, vector_store, model_type)
+    # Process the query with the reference for evaluation
+    response, source_text, context_text, metrics_text, language = process_query(query, reference)
     
-    return "Successfully initialized with mock Vision 2030 documents. The assistant is ready for testing!"
-
-@spaces.GPU
-def answer_query(message, history):
-    global assistant
+    # Add reference to the output
+    reference_text = f"""
+## Reference Answer:
+{reference}
+    """
     
-    if assistant is None:
-        return [(message, "Please load a model and process documents first (or use mock documents for testing).")]
-    
-    response = assistant.answer(message)
-    history.append((message, response))
-    return history
+    return response, source_text, context_text, metrics_text + reference_text, language
 
 def reset_chat():
-    global assistant
-    
-    if assistant is None:
-        return "No active conversation to reset."
-    
-    reset_message = assistant.reset_conversation()
-    return reset_message
+    """Reset the conversation history"""
+    if ASSISTANT:
+        ASSISTANT.reset_conversation()
+        return "Conversation has been reset."
+    return "System not initialized."
 
-def restart_factory():
-    return "Restarting the application... Please reload the page in a few seconds."
+def qualitative_feedback(response, user_feedback, feedback_type):
+    """Save qualitative feedback from users"""
+    try:
+        feedback_data = {
+            "response": response,
+            "user_feedback": user_feedback,
+            "feedback_type": feedback_type,
+            "timestamp": str(datetime.datetime.now())
+        }
+        
+        # Ensure directory exists
+        os.makedirs("feedback", exist_ok=True)
+        
+        # Append to feedback file
+        with open("feedback/user_feedback.jsonl", "a") as f:
+            f.write(json.dumps(feedback_data) + "\n")
+        
+        return f"Thank you for your {feedback_type} feedback!"
+    except Exception as e:
+        return f"Error saving feedback: {e}"
 
 # Create Gradio interface
-with gr.Blocks(title="Vision 2030 Virtual Assistant") as demo:
-    gr.Markdown("# Vision 2030 Virtual Assistant")
-    gr.Markdown("Ask questions about Saudi Vision 2030 goals, projects, and progress in Arabic or English.")
+with gr.Blocks(title="Vision 2030 Assistant - Qualitative Evaluation") as demo:
+    gr.Markdown("# Vision 2030 Virtual Assistant - Qualitative Evaluation")
+    gr.Markdown("This interface allows you to interact with and evaluate the multilingual Vision 2030 Assistant.")
     
-    with gr.Tab("Setup"):
-        gr.Markdown("## Step 1: Load a Model")
-        with gr.Row():
-            with gr.Column():
-                primary_btn = gr.Button("Load ALLaM-7B Model (Primary)", variant="primary")
-                primary_output = gr.Textbox(label="Primary Model Status")
-                primary_btn.click(load_primary_model, inputs=[], outputs=primary_output)
-            
-            with gr.Column():
-                fallback_btn = gr.Button("Load BLOOM-7B1 (Fallback)", variant="secondary")
-                fallback_output = gr.Textbox(label="Fallback Model Status")
-                fallback_btn.click(load_fallback_model, inputs=[], outputs=fallback_output)
-            
-            with gr.Column():
-                mbart_btn = gr.Button("Load mBART (Alternative)", variant="secondary")
-                mbart_output = gr.Textbox(label="mBART Model Status")
-                mbart_btn.click(load_mbart_model, inputs=[], outputs=mbart_output)
+    with gr.Tab("System Initialization"):
+        init_button = gr.Button("Initialize System")
+        init_output = gr.Textbox(label="Initialization Status")
         
-        gr.Markdown("## Step 2: Prepare Documents")
-        with gr.Row():
-            with gr.Column():
-                pdf_files = gr.File(file_types=[".pdf"], file_count="multiple", label="Upload PDF Documents")
-                process_btn = gr.Button("Process Documents", variant="primary")
-                process_output = gr.Textbox(label="Processing Status")
-                process_btn.click(process_pdfs, inputs=[pdf_files], outputs=process_output)
-            
-            with gr.Column():
-                mock_btn = gr.Button("Use Mock Documents (for testing)", variant="secondary")
-                mock_output = gr.Textbox(label="Mock Documents Status")
-                mock_btn.click(use_mock_documents, inputs=[], outputs=mock_output)
-        
-        gr.Markdown("## Troubleshooting")
-        restart_btn = gr.Button("Restart Application", variant="secondary")
-        restart_output = gr.Textbox(label="Restart Status")
-        restart_btn.click(restart_factory, inputs=[], outputs=restart_output)
-        restart_btn.click(None, [], None, _js="() => {setTimeout(() => {location.reload()}, 5000)}")
+        init_button.click(initialize_system, inputs=[], outputs=[init_output])
     
-    with gr.Tab("Chat"):
-        chatbot = gr.Chatbot(label="Conversation", height=500)
-        
+    with gr.Tab("Chat & Evaluation"):
         with gr.Row():
-            message = gr.Textbox(
-                label="Ask a question about Vision 2030 (in Arabic or English)",
-                placeholder="What are the main goals of Vision 2030?",
-                lines=2
-            )
-            submit_btn = gr.Button("Submit", variant="primary")
-        
-        reset_btn = gr.Button("Reset Conversation")
-        
-        gr.Markdown("### Example Questions")
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("**English Questions:**")
-                en_examples = gr.Examples(
-                    examples=[
-                        "What is Saudi Vision 2030?",
-                        "What are the economic goals of Vision 2030?",
-                        "How does Vision 2030 support women's empowerment?",
-                        "What environmental initiatives are part of Vision 2030?",
-                        "What is the role of the Public Investment Fund in Vision 2030?"
-                    ],
-                    inputs=message
-                )
-            
-            with gr.Column():
-                gr.Markdown("**Arabic Questions:**")
-                ar_examples = gr.Examples(
-                    examples=[
-                        "ما هي رؤية السعودية 2030؟",
-                        "ما هي الأهداف الاقتصادية لرؤية 2030؟",
-                        "كيف تدعم رؤية 2030 تمكين المرأة السعودية؟",
-                        "ما هي مبادرات رؤية 2030 للحفاظ على البيئة؟",
-                        "ما هي استراتيجية صندوق الاستثمارات العامة في رؤية 2030؟"
-                    ],
-                    inputs=message
-                )
-        
-        reset_output = gr.Textbox(label="Reset Status", visible=False)
-        submit_btn.click(answer_query, inputs=[message, chatbot], outputs=[chatbot])
-        message.submit(answer_query, inputs=[message, chatbot], outputs=[chatbot])
-        reset_btn.click(reset_chat, inputs=[], outputs=[reset_output])
-        reset_btn.click(lambda: None, inputs=[], outputs=[chatbot], postprocess=lambda: [])
+            with gr.Column(scale=2):
+                query_input = gr.Textbox(label="Ask about Saudi Vision 2030 (in English or Arabic)", lines=3)
+                reference_input = gr.Textbox(label="Reference Answer (Optional - for evaluation)", lines=3)
+                
+                with gr.Row():
+                    submit_btn = gr.Button("Submit")
+                    reset_btn = gr.Button("Reset Chat")
+                
+                response_output = gr.Textbox(label="Response", lines=6)
+                
+                with gr.Accordion("Evaluation Metrics", open=False):
+                    metrics_output = gr.Markdown()
+                
+                with gr.Accordion("Retrieved Sources", open=False):
+                    sources_output = gr.Textbox(label="Sources")
+                
+                with gr.Accordion("Retrieved Contexts", open=False):
+                    contexts_output = gr.Textbox(label="Contexts", lines=10)
+                
+                with gr.Accordion("Qualitative Feedback", open=False):
+                    feedback_text = gr.Textbox(label="Your Feedback", lines=3)
+                    feedback_type = gr.Radio(
+                        ["Correctness", "Relevance", "Fluency", "Completeness", "Other"],
+                        label="Feedback Type"
+                    )
+                    feedback_btn = gr.Button("Submit Feedback")
+                    feedback_output = gr.Textbox(label="Feedback Status")
+    
+    with gr.Tab("Sample Evaluation"):
+        sample_index = gr.Slider(0, len(sample_evaluation_data)-1, 0, step=1, label="Sample Index")
+        eval_btn = gr.Button("Evaluate Sample")
+        
+        sample_response = gr.Textbox(label="Response", lines=6)
+        sample_metrics = gr.Markdown(label="Metrics & Reference")
+        
+        with gr.Accordion("Retrieved Sources", open=False):
+            sample_sources = gr.Textbox(label="Sources")
+        
+        with gr.Accordion("Retrieved Contexts", open=False):
+            sample_contexts = gr.Textbox(label="Contexts", lines=10)
+    
+    with gr.Tab("About"):
+        gr.Markdown("""
+        ## Vision 2030 Assistant
+        
+        This is a multilingual RAG-based Conversational Agent using ALLaM-7B for answering questions about Saudi Vision 2030.
+        
+        ### Features:
+        - Supports both Arabic and English queries
+        - Uses Retrieval-Augmented Generation (RAG) for accurate answers
+        - Provides transparent sources for information
+        - Comprehensive evaluation metrics
+        
+        ### How to use:
+        1. Initialize the system (first tab)
+        2. Ask questions about Saudi Vision 2030 in the Chat tab
+        3. Optionally provide reference answers for evaluation
+        4. Explore sample evaluations from our test dataset
+        
+        ### Evaluation Metrics:
+        - ROUGE: Measures overlap of n-grams between response and reference
+        - BLEU: Measures precision of n-grams in the response compared to reference
+        - METEOR: Measures semantic similarity between response and reference
+        - F1/Precision/Recall: Word-level comparison metrics
+        """)
+    
+    # Set up event handlers
+    submit_btn.click(
+        process_query, 
+        inputs=[query_input, reference_input], 
+        outputs=[response_output, sources_output, contexts_output, metrics_output]
+    )
+    
+    reset_btn.click(
+        reset_chat,
+        inputs=[],
+        outputs=[response_output]
+    )
+    
+    eval_btn.click(
+        evaluate_sample,
+        inputs=[sample_index],
+        outputs=[sample_response, sample_sources, sample_contexts, sample_metrics]
+    )
+    
+    feedback_btn.click(
+        qualitative_feedback,
+        inputs=[response_output, feedback_text, feedback_type],
+        outputs=[feedback_output]
+    )
 
-# Launch the app
-demo.launch()
+# Launch the interface
+if __name__ == "__main__":
+    demo.launch()