Upload 11 files

README

@@ -0,0 +1,49 @@
+
+# ArabicRAG: Arabic Retrieval-Augmented Generation
+
+### Project Structure
+# arabic_legal_search/
+# ├── config.py
+# ├── document_processor.py
+# ├── embedding_generator.py
+# ├── search_engine.py
+# ├── response_generator.py
+# ├── utils.py
+# ├── main.py
+# └── requirements.txt
+## Overview
+ArabicRAG is an open-source project designed to leverage the power of retrieval-augmented generation for processing and understanding Arabic legal documents. The system integrates advanced NLP techniques to retrieve relevant documents and generate context-aware responses.
+
+## Features
+- **Document Processing**: Load and preprocess Arabic text documents efficiently.
+- **Embedding Generation**: Utilize multilingual models to generate embeddings for Arabic text.
+- **Efficient Search**: Leverage FAISS for fast and efficient similarity search in large document corpora.
+- **Response Generation**: Use state-of-the-art transformer models to generate responses based on retrieved context.
+
+## Installation
+To set up your environment and run ArabicRAG, follow these steps:
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/maljefairi/arabicRAG
+   ```
+2. Install the required packages:
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+## Usage
+After installation, you can run the main script to start processing documents:
+```bash
+python main.py
+```
+
+## Contributing
+Contributions are welcome! For major changes, please open an issue first to discuss what you would like to change. Please make sure to update tests as appropriate.
+
+## License
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## Contact
+- **Dr. Mohammed Al-Jefairi** - maljefairi@sidramail.com
+- **GitHub**: [maljefairi](https://github.com/maljefairi/arabicRAG)

config.py

@@ -0,0 +1,10 @@
+# config.py
+import os
+
+class Config:
+    DOCUMENT_FOLDER = os.environ.get('DOCUMENT_FOLDER', 'data')    EMBEDDING_MODEL = os.environ.get('EMBEDDING_MODEL', 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2')
+    LLM_MODEL = os.environ.get('LLM_MODEL', 'CAMeL-Lab/bert-base-arabic-camelbert-ca')
+    BATCH_SIZE = int(os.environ.get('BATCH_SIZE', 32))
+    TOP_K = int(os.environ.get('TOP_K', 5))
+    MAX_LENGTH = int(os.environ.get('MAX_LENGTH', 1024))
+    

document_processor.py

@@ -0,0 +1,19 @@
+# document_processor.py
+import os
+import glob
+from tqdm import tqdm
+import pandas as pd
+from utils import clean_text, setup_logger
+
+logger = setup_logger('document_processor')
+
+def load_documents(folder_path):
+    documents = []
+    for file_path in tqdm(glob.glob(os.path.join(folder_path, '*.txt')), desc="Loading documents"):
+        try:
+            with open(file_path, 'r', encoding='utf-8') as file:
+                content = clean_text(file.read())
+                documents.append({'path': file_path, 'content': content})
+        except Exception as e:
+            logger.error(f"Error reading {file_path}: {e}")
+    return pd.DataFrame(documents)

embedding_generator.py

@@ -0,0 +1,20 @@
+# embedding_generator.py
+import numpy as np
+from tqdm import tqdm
+from sentence_transformers import SentenceTransformer
+from utils import setup_logger
+from config import Config
+
+logger = setup_logger('embedding_generator')
+
+def generate_embeddings(documents):
+    model = SentenceTransformer(Config.EMBEDDING_MODEL)
+    embeddings = []
+    for i in tqdm(range(0, len(documents), Config.BATCH_SIZE), desc="Generating embeddings"):
+        batch = documents['content'][i:i+Config.BATCH_SIZE].tolist()
+        try:
+            batch_embeddings = model.encode(batch, show_progress_bar=False)
+            embeddings.extend(batch_embeddings)
+        except Exception as e:
+            logger.error(f"Error encoding batch: {e}")
+    return np.array(embeddings)

main.py

@@ -0,0 +1,79 @@
+# main.py
+import os
+import sys
+from document_processor import load_documents
+from embedding_generator import generate_embeddings
+from search_engine import SearchEngine
+from response_generator import ResponseGenerator
+from config import Config
+from utils import setup_logger
+
+logger = setup_logger('main')
+
+def initialize_system():
+    """Initialize the search and response system."""
+    logger.info("Initializing the system...")
+    
+    # Load and process documents
+    documents = load_documents(Config.DOCUMENT_FOLDER)
+    logger.info(f"Loaded {len(documents)} documents")
+
+    # Generate embeddings
+    embeddings = generate_embeddings(documents)
+    logger.info(f"Generated embeddings of shape {embeddings.shape}")
+
+    # Initialize search engine
+    search_engine = SearchEngine(documents, embeddings)
+    logger.info("Search engine initialized")
+
+    # Initialize response generator
+    response_generator = ResponseGenerator()
+    logger.info("Response generator initialized")
+
+    return search_engine, response_generator
+
+def process_query(query, search_engine, response_generator):
+    """Process a single query and return the response."""
+    relevant_docs = search_engine.search(query)
+    logger.info(f"Found {len(relevant_docs)} relevant documents")
+
+    response = response_generator.generate_response(query, relevant_docs)
+    return response
+
+def interactive_mode(search_engine, response_generator):
+    """Run the system in interactive mode, processing queries from user input."""
+    print("Enter your queries. Type 'quit' to exit.")
+    while True:
+        query = input("Query: ").strip()
+        if query.lower() == 'quit':
+            break
+        
+        response = process_query(query, search_engine, response_generator)
+        print(f"Response: {response}\n")
+
+def batch_mode(input_file, output_file, search_engine, response_generator):
+    """Process queries from an input file and write responses to an output file."""
+    with open(input_file, 'r', encoding='utf-8') as infile, open(output_file, 'w', encoding='utf-8') as outfile:
+        for line in infile:
+            query = line.strip()
+            response = process_query(query, search_engine, response_generator)
+            outfile.write(f"Query: {query}\nResponse: {response}\n\n")
+    logger.info(f"Batch processing completed. Results written to {output_file}")
+
+def main():
+    search_engine, response_generator = initialize_system()
+
+    if len(sys.argv) > 1:
+        if sys.argv[1] == '--batch':
+            if len(sys.argv) != 4:
+                print("Usage for batch mode: python main.py --batch input_file output_file")
+                sys.exit(1)
+            batch_mode(sys.argv[2], sys.argv[3], search_engine, response_generator)
+        else:
+            print("Unknown argument. Use --batch for batch mode or no arguments for interactive mode.")
+            sys.exit(1)
+    else:
+        interactive_mode(search_engine, response_generator)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,26 @@
+# requirements.txt
+# Standard libraries for data handling and computation
+numpy==1.26.4
+pandas==1.3.5
+scipy==1.14.1
+
+# Machine Learning and NLP libraries
+torch==2.4.0+cu116  # Specify CUDA version if needed
+torchvision==0.19.0+cu116  # Specify CUDA version if needed
+transformers==4.15.0
+sentence-transformers==2.2.0
+
+# For efficient nearest neighbor search
+faiss-gpu==1.8.0  # Make sure to use the GPU version for CUDA compatibility
+
+# Utilities
+tqdm==4.62.3
+setuptools_rust  # Required for compiling tokenizers with Rust dependencies
+tokenizers==0.10.3  # Ensure compatibility with transformers
+
+# If you're using Rust-based libraries
+rust==2024.1  # Pseudo-version, adjust based on your need or omit if not using Rust directly
+
+# Additional Python packages that may be required
+huggingface-hub==0.24.6
+streamlit

response_generator.py

@@ -0,0 +1,73 @@
+# response_generator.py
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from utils import setup_logger
+from config import Config
+
+logger = setup_logger('response_generator')
+
+class ResponseGenerator:
+    def __init__(self):
+        self.tokenizer = AutoTokenizer.from_pretrained(Config.LLM_MODEL)
+        self.model = AutoModelForCausalLM.from_pretrained(Config.LLM_MODEL)
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.model.to(self.device)
+        logger.info(f"Model loaded and moved to {self.device}")
+
+    def generate_response(self, query, relevant_docs):
+        try:
+            context = self._prepare_context(relevant_docs)
+            prompt = self._create_prompt(query, context)
+            input_ids = self.tokenizer.encode(prompt, return_tensors="pt").to(self.device)
+            
+            attention_mask = input_ids.ne(self.tokenizer.pad_token_id).float()
+            
+            with torch.no_grad():
+                output = self.model.generate(
+                    input_ids,
+                    attention_mask=attention_mask,
+                    max_length=Config.MAX_LENGTH,
+                    num_return_sequences=1,
+                    no_repeat_ngram_size=2,
+                    do_sample=True,
+                    top_k=50,
+                    top_p=0.95,
+                    temperature=0.7
+                )
+            
+            response = self.tokenizer.decode(output[0], skip_special_tokens=True)
+            return self._extract_answer(response)
+        
+        except Exception as e:
+            logger.error(f"Error generating response: {e}")
+            return "عذرًا، لم أتمكن من إنشاء استجابة بسبب خطأ ما."  # "Sorry, I couldn't generate a response due to an error."
+
+    def _prepare_context(self, relevant_docs):
+        # Combine content from relevant documents
+        combined_content = "\n".join(relevant_docs['content'].tolist())
+        # Truncate if too long
+        max_context_length = Config.MAX_LENGTH // 2  # Use half of max_length for context
+        return combined_content[:max_context_length]
+
+    def _create_prompt(self, query, context):
+        return f"""مستند قانوني:
+{context}
+
+سؤال:
+{query}
+
+إجابة:"""
+
+    def _extract_answer(self, response):
+        # Extract the generated answer from the full response
+        answer_start = response.find("إجابة:") + len("إجابة:")
+        return response[answer_start:].strip()
+
+    def update_model(self, new_model_name):
+        try:
+            self.tokenizer = AutoTokenizer.from_pretrained(new_model_name)
+            self.model = AutoModelForCausalLM.from_pretrained(new_model_name)
+            self.model.to(self.device)
+            logger.info(f"Model updated to {new_model_name}")
+        except Exception as e:
+            logger.error(f"Error updating model: {e}")

search_engine.py

@@ -0,0 +1,30 @@
+# search_engine.py
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from utils import setup_logger
+from config import Config
+
+logger = setup_logger('search_engine')
+
+class SearchEngine:
+    def __init__(self, documents, embeddings):
+        self.documents = documents
+        self.index = self._build_faiss_index(embeddings)
+        self.model = SentenceTransformer(Config.EMBEDDING_MODEL)
+
+    def _build_faiss_index(self, embeddings):
+        dimension = embeddings.shape[1]
+        index = faiss.IndexFlatL2(dimension)
+        index.add(embeddings.astype('float32'))
+        return index
+
+    def search(self, query):
+        try:
+            query_embedding = self.model.encode([query])
+            _, indices = self.index.search(query_embedding.astype('float32'), Config.TOP_K)
+            return self.documents.iloc[indices[0]]
+        except Exception as e:
+            logger.error(f"Error searching documents: {e}")
+            return pd.DataFrame()
+

streamlit_app.py

@@ -0,0 +1,63 @@
+import streamlit as st
+from main import initialize_system, process_query
+
+# ---------------------------------------------------------
+# Streamlit page configuration
+# ---------------------------------------------------------
+st.set_page_config(
+    page_title="Arabic RAG Chatbot 🤖",
+    page_icon="🤖",
+    layout="wide",
+)
+
+# ---------------------------------------------------------
+# Title and description
+# ---------------------------------------------------------
+st.title("🤖 Arabic RAG Chatbot")
+st.markdown("""
+مرحبًا! 👋  
+اكتب سؤالك بالعربية وسيتولى النظام الإجابة استنادًا إلى مستنداتك المخزّنة محليًا.
+""")
+
+# ---------------------------------------------------------
+# Cached system initialization (so it doesn't reload every time)
+# ---------------------------------------------------------
+@st.cache_resource
+def load_rag_system():
+    search_engine, response_generator = initialize_system()
+    return search_engine, response_generator
+
+search_engine, response_generator = load_rag_system()
+
+# ---------------------------------------------------------
+# Input section
+# ---------------------------------------------------------
+st.divider()
+query = st.text_input("📝 أدخل سؤالك هنا:", placeholder="مثال: ما هي نسبة الحضور المطلوبة؟")
+
+# ---------------------------------------------------------
+# Query handling
+# ---------------------------------------------------------
+if st.button("بحث") or query:
+    if not query.strip():
+        st.warning("يرجى كتابة سؤال أولاً.")
+    else:
+        with st.spinner("⏳ جارٍ البحث عن الإجابة..."):
+            try:
+                response = process_query(query, search_engine, response_generator)
+                if response:
+                    st.success("💬 الإجابة:")
+                    st.write(response)
+                else:
+                    st.info("لم يتم العثور على إجابة ذات صلة في المستندات.")
+            except Exception as e:
+                st.error(f"حدث خطأ أثناء توليد الإجابة: {e}")
+
+# ---------------------------------------------------------
+# Footer
+# ---------------------------------------------------------
+st.divider()
+st.markdown(
+    "<p style='text-align:center; color:gray;'>تم التطوير باستخدام Streamlit و RAG ❤️</p>",
+    unsafe_allow_html=True
+)

utils.py

@@ -0,0 +1,19 @@
+# utils.py
+import re
+import logging
+
+def clean_text(text):
+    # Remove special characters and multiple spaces
+    text = re.sub(r'[^\w\s\u0600-\u06FF]', ' ', text)
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def setup_logger(name):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+    handler = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    return logger
+