inital

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,17 @@
 ---
-title: RAG1
-emoji: 🚀
-colorFrom: pink
-colorTo: gray
+title: RAG
+emoji: 💬
+colorFrom: yellow
+colorTo: purple
 sdk: gradio
-sdk_version: 6.13.0
+sdk_version: 6.5.1
 app_file: app.py
 pinned: false
+hf_oauth: true
+hf_oauth_scopes:
+- inference-api
+license: apache-2.0
+short_description: 'RAG-Palestine '
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+An example chatbot using [Gradio](https://gradio.app), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/v0.22.2/en/index), and the [Hugging Face Inference API](https://huggingface.co/docs/api-inference/index).

app.py

@@ -0,0 +1,429 @@
+import gradio as gr
+import json
+import os
+from rag_pipeline import (
+    initialise_pipeline,
+    add_pdf_to_index,
+    query_rag,
+    summarise_document,
+    compare_documents,
+    analyse_discourse
+)
+from extensions import (
+    generate_map,
+    generate_timeline,
+    generate_wordcloud,
+    get_statistics,
+    advanced_analytics,
+    text_to_speech,
+    speech_to_text,
+    translate_text,
+    export_chat_history
+)
+
+# Initialize the pipeline
+initialise_pipeline()
+
+MODELS = [
+    "gpt-oss-120b",
+    "google/gemma-4-31B",
+    "openrouter/auto"
+]
+
+def get_document_list():
+    """Read document filenames from disk — always fresh, supports Unicode/Arabic names."""
+    docs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "documents")
+    if not os.path.exists(docs_dir):
+        return []
+    return sorted(
+        [f for f in os.listdir(docs_dir) if f.lower().endswith(".pdf")],
+        key=lambda x: x.lower()
+    )
+
+def refresh_doc_dropdowns():
+    """Return gr.update() calls for all document dropdowns."""
+    docs = get_document_list()
+    return (
+        gr.update(choices=docs),
+        gr.update(choices=docs),
+        gr.update(choices=docs),
+        gr.update(choices=["All"] + docs),
+    )
+
+def respond_advanced(message, audio_path, history, model_id):
+    if audio_path and not message:
+        message = speech_to_text(audio_path)
+    
+    if not message:
+        return "", history
+        
+    answer, sources = query_rag(message, model_id=model_id)
+    
+    if sources:
+        source_text = "\n\n**Sources:**\n"
+        for i, s in enumerate(sources):
+            source_text += f"- **{s['source']}** (Page {s['page']}, Score: {s['score']})\n"
+        answer += source_text
+        
+    history.append({"role": "user", "content": message})
+    history.append({"role": "assistant", "content": answer})
+    return "", history
+
+def clear_chat():
+    return []
+
+def toggle_translation(history):
+    if not history:
+        return history
+    last_msg = history[-1]
+    last_bot_raw = last_msg["content"] if isinstance(last_msg, dict) else getattr(last_msg, "content", "")
+    
+    # In Gradio 5.x, content can sometimes be a tuple or list for multimodal messages.
+    if isinstance(last_bot_raw, (list, tuple)):
+        last_bot = last_bot_raw[0] if len(last_bot_raw) > 0 else ""
+        if isinstance(last_bot, dict) and "text" in last_bot:
+            last_bot = last_bot["text"]
+        elif hasattr(last_bot, "text"):
+            last_bot = last_bot.text
+    else:
+        last_bot = str(last_bot_raw)
+        
+    # Separate the answer from the sources to avoid translating metadata and hitting 5000 char limits
+    parts = last_bot.split("\n\n**Sources:**\n")
+    main_text = parts[0]
+    sources_text = "\n\n**Sources:**\n" + parts[1] if len(parts) > 1 else ""
+    
+    has_arabic = any("\u0600" <= c <= "\u06FF" for c in main_text)
+    target_lang = 'en' if has_arabic else 'ar'
+    
+    # Translate only the main text
+    translated_main = translate_text(main_text, target_lang=target_lang)
+    translated_full = translated_main + sources_text
+    
+    if isinstance(last_msg, dict):
+        history[-1]["content"] = translated_full
+    else:
+        history[-1].content = translated_full
+        
+    return list(history)
+
+def get_audio_for_last_response(history):
+    if not history:
+        return None
+    last_msg = history[-1]
+    last_bot_raw = last_msg["content"] if isinstance(last_msg, dict) else getattr(last_msg, "content", "")
+    
+    if isinstance(last_bot_raw, (list, tuple)):
+        last_bot = last_bot_raw[0] if len(last_bot_raw) > 0 else ""
+        if isinstance(last_bot, dict) and "text" in last_bot:
+            last_bot = last_bot["text"]
+        elif hasattr(last_bot, "text"):
+            last_bot = last_bot.text
+    else:
+        last_bot = str(last_bot_raw)
+        
+    # Safely strip out sources block
+    main_text = last_bot.split("\n\n**Sources:**\n")[0]
+    clean_text = main_text.replace('*', '').replace('#', '')
+    
+    has_arabic = any("\u0600" <= c <= "\u06FF" for c in clean_text)
+    lang = 'ar' if has_arabic else 'en'
+    return text_to_speech(clean_text, lang=lang)
+
+PALESTINE_CSS = """
+/* Palestinian Theme */
+@import url('https://fonts.googleapis.com/css2?family=Cairo:wght@400;600;700&family=Inter:wght@400;500;600&display=swap');
+
+/* Keffiyeh/Checker Background */
+body, .gradio-container, .main, .wrap {
+    background-color: #0B120E !important;
+    background-image: url("data:image/svg+xml,%3Csvg width='40' height='40' viewBox='0 0 40 40' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath d='M0 20 L20 0 L40 20 L20 40 Z' fill='none' stroke='%23ffffff' stroke-opacity='0.07' stroke-width='1'/%3E%3C/svg%3E") !important;
+    background-repeat: repeat !important;
+    background-attachment: fixed !important;
+    font-family: 'Inter', 'Cairo', sans-serif !important;
+    color: #e8f5e8 !important;
+}
+
+/* Palestinian Flag Top Border */
+.gradio-container {
+    border-top: 12px solid !important;
+    border-image: linear-gradient(to right, #000000 33%, #FFFFFF 33%, #FFFFFF 66%, #007A3D 66%) 1 !important;
+}
+
+/* Chatbot Container */
+#pali-chatbot {
+    background: rgba(18, 28, 22, 0.7) !important;
+    border: 2px solid #007A3D !important;
+}
+
+/* Tabs */
+.tab-nav button {
+    background: #0B120E !important;
+    color: #9dc99d !important;
+    border-bottom: 2px solid transparent !important;
+    font-weight: 500;
+    transition: all 0.2s ease;
+}
+.tab-nav button.selected {
+    color: #ffffff !important;
+    border-bottom: 3px solid #007A3D !important;
+    background: rgba(0,122,61,0.12) !important;
+}
+.tab-nav button:hover {
+    color: #ffffff !important;
+    background: rgba(0,122,61,0.08) !important;
+}
+.tab-nav {
+    border-bottom: 2px solid #CE1126 !important;
+}
+
+/* Primary Buttons */
+button.primary, .gr-button-primary, button[variant="primary"] {
+    background: linear-gradient(135deg, #007A3D, #009e50) !important;
+    border: none !important;
+    color: white !important;
+    font-weight: 600;
+    transition: opacity 0.2s;
+}
+button.primary:hover, button[variant="primary"]:hover { opacity: 0.88; }
+
+/* Secondary Buttons */
+button.secondary, .gr-button-secondary, button[variant="secondary"] {
+    background: #0a0f0a !important;
+    border: 1px solid #007A3D !important;
+    color: #7ecf7e !important;
+}
+
+/* Inputs, Textareas, and Dropdowns */
+textarea, input[type=text], input[type=search], select, div.wrap-inner, div[role="listbox"], div[role="combobox"] {
+    background-color: #000000 !important;
+    border: 1px solid #A49966 !important;
+    color: #ffffff !important;
+    caret-color: #007A3D;
+}
+textarea:focus, input[type=text]:focus, input[type=search]:focus {
+    border-color: #007A3D !important;
+    outline: none !important;
+    box-shadow: 0 0 0 2px rgba(0,122,61,0.25) !important;
+}
+
+/* Chatbot Message Bubbles */
+.message.user { background: rgba(0,122,61,0.20) !important; }
+.message.bot  { background: rgba(30,61,30,0.50) !important; }
+
+/* Labels */
+label span, .gr-form label, label > span {
+    color: #9dc99d !important;
+    font-weight: 600;
+    font-size: 0.85rem;
+    letter-spacing: 0.03em;
+}
+
+/* Dataframes */
+.dataframe th { background: #007A3D !important; color: white !important; }
+.dataframe tr:nth-child(even) { background: #0f1a10 !important; }
+.dataframe tr:nth-child(odd)  { background: #111c12 !important; }
+
+/* Scrollbars */
+::-webkit-scrollbar { width: 6px; height: 6px; }
+::-webkit-scrollbar-track { background: #0B120E; }
+::-webkit-scrollbar-thumb { background: #007A3D; border-radius: 3px; }
+::-webkit-scrollbar-thumb:hover { background: #009e50; }
+
+/* Title glow */
+h1 { text-shadow: 0 0 20px rgba(0,122,61,0.4); }
+
+footer { visibility: hidden !important; }
+"""
+
+PROFILE_PIC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "profile_picture.png")
+
+with gr.Blocks(title="Palestinian Agentic RAG Platform") as demo:
+    gr.Markdown(
+        """
+        <div style="display:flex;align-items:center;gap:16px;padding:12px 0;">
+            <img src="/file=profile_picture.png" style="width:70px;height:70px;border-radius:50%;border:3px solid #007A3D;" />
+            <div>
+                <h1 style="margin:0;font-size:1.6rem;color:#ffffff;font-family:'Cairo',sans-serif;"> AI for a Free Palestine </h1>
+                <p style="margin:0;color:#9dc99d;font-size:0.9rem;">   Ask . Learn . Understand — in support of a free Palestine.</p>
+            </div>
+        </div>
+        """
+    )
+    
+    with gr.Tabs():
+        # 1. SMART CHAT
+        with gr.TabItem("💬 Smart Chat"):
+            model_chat = gr.Dropdown(choices=MODELS, value="gpt-oss-120b", label="Select Model")
+            chatbot = gr.Chatbot(
+                label="Agentic RAG Chatbot",
+                height=520,
+                avatar_images=(None, PROFILE_PIC),
+                elem_id="pali-chatbot",
+            )
+            with gr.Row():
+                with gr.Column(scale=8):
+                    msg = gr.Textbox(label="Type your question here...", placeholder="What is the history of the Balfour Declaration?")
+                with gr.Column(scale=1):
+                    audio_in = gr.Audio(sources=["microphone", "upload"], type="filepath", label="Voice Input")
+            
+            with gr.Row():
+                submit_btn = gr.Button("Submit", variant="primary")
+                clear_btn = gr.Button("Clear Chat")
+                
+            with gr.Row():
+                translate_btn = gr.Button("🔄 Auto-Translate Last Answer")
+                speak_btn = gr.Button("🔊 Speak Last Answer")
+                export_btn = gr.Button("💾 Export Chat")
+                
+            audio_out = gr.Audio(label="Voice Output", interactive=False)
+            export_file = gr.File(label="Download Chat Export", interactive=False)
+            
+            submit_btn.click(respond_advanced, inputs=[msg, audio_in, chatbot, model_chat], outputs=[msg, chatbot])
+            msg.submit(respond_advanced, inputs=[msg, audio_in, chatbot, model_chat], outputs=[msg, chatbot])
+            clear_btn.click(clear_chat, outputs=chatbot)
+            
+            translate_btn.click(toggle_translation, inputs=chatbot, outputs=chatbot)
+            speak_btn.click(get_audio_for_last_response, inputs=chatbot, outputs=audio_out)
+            export_btn.click(export_chat_history, inputs=chatbot, outputs=export_file)
+            
+        # 2. DISCOURSE ANALYSIS
+        with gr.TabItem("🔍 Discourse Analysis"):
+            gr.Markdown("### Detect bias, framing, and text orientation")
+            topic_input = gr.Textbox(label="Topic or Text to Analyze")
+            model_discourse = gr.Dropdown(choices=MODELS, value="gpt-oss-120b", label="Select Model")
+            analyse_btn = gr.Button("Analyze Discourse")
+            analyse_output = gr.Textbox(label="Discourse Analysis Report", lines=15)
+            analyse_btn.click(fn=analyse_discourse, inputs=[topic_input, model_discourse], outputs=analyse_output)
+
+        # 3. COMPARE DOCUMENTS
+        with gr.TabItem("⚖️ Compare Documents"):
+            gr.Markdown("### Side-by-side comparison of two documents or topics")
+            with gr.Row():
+                doc1_input = gr.Dropdown(choices=get_document_list(), label="Document 1", allow_custom_value=True)
+                doc2_input = gr.Dropdown(choices=get_document_list(), label="Document 2", allow_custom_value=True)
+            aspect_input = gr.Textbox(label="Aspect to compare", value="Main themes and biases")
+            model_compare_docs = gr.Dropdown(choices=MODELS, value="gpt-oss-120b", label="Select Model")
+            compare_btn = gr.Button("Compare")
+            compare_output = gr.Textbox(label="Comparison Result", lines=15)
+            compare_btn.click(fn=compare_documents, inputs=[doc1_input, doc2_input, aspect_input, model_compare_docs], outputs=compare_output)
+
+        # 4. DOCUMENT SUMMARY
+        with gr.TabItem("📝 Document Summary"):
+            gr.Markdown("### Auto-summarize any document")
+            doc_name_input = gr.Dropdown(choices=get_document_list(), label="Select Document", allow_custom_value=True)
+            model_summary = gr.Dropdown(choices=MODELS, value="gpt-oss-120b", label="Select Model")
+            summarise_btn = gr.Button("Generate Summary")
+            summarise_output = gr.Textbox(label="Summary", lines=10)
+            summarise_btn.click(fn=summarise_document, inputs=[doc_name_input, model_summary], outputs=summarise_output)
+
+        # 5. INTERACTIVE MAP
+        with gr.TabItem("🗺️ Interactive Map"):
+            gr.Markdown("### Palestinian historical locations and context")
+            map_btn = gr.Button("Load Interactive Map")
+            map_html = gr.HTML(label="Map View")
+            
+            def load_map_iframe():
+                path = generate_map()
+                with open(path, "r", encoding="utf-8") as f:
+                    html_data = f.read()
+                escaped_html = html_data.replace('"', '&quot;')
+                return f'<iframe srcdoc="{escaped_html}" width="100%" height="600px" style="border:none;"></iframe>'
+                    
+            map_btn.click(load_map_iframe, outputs=map_html)
+
+        # 6. HISTORICAL TIMELINE
+        with gr.TabItem("⏳ Historical Timeline"):
+            gr.Markdown("### Key Events Timeline")
+            timeline_html = gr.HTML(value=generate_timeline())
+
+        # 7. WORD CLOUD
+        with gr.TabItem("☁️ Word Cloud"):
+            gr.Markdown("### Word frequency visualization")
+            wc_doc_name = gr.Dropdown(choices=["All"] + get_document_list(), value="All", label="Select Document (or 'All' for entire corpus)", allow_custom_value=True)
+            wc_btn = gr.Button("Generate Word Cloud")
+            wc_img = gr.Image(label="Word Cloud")
+            wc_btn.click(fn=generate_wordcloud, inputs=wc_doc_name, outputs=wc_img)
+
+        # 8. STATISTICS
+        with gr.TabItem("📊 Statistics"):
+            gr.Markdown("### Corpus metrics and distribution")
+            stats_btn = gr.Button("Refresh Statistics")
+            with gr.Row():
+                stats_summary = gr.Dataframe(label="Document Summaries")
+                stats_raw = gr.Dataframe(label="Raw Chunks")
+            stats_btn.click(fn=get_statistics, outputs=[stats_summary, stats_raw])
+
+        # 9. UPLOAD PDF
+        with gr.TabItem("📄 Upload PDF"):
+            gr.Markdown("### Add a new PDF to the index instantly")
+            file_input = gr.File(label="Upload PDF", file_types=[".pdf"])
+            with gr.Row():
+                upload_btn = gr.Button("Add to Index", variant="primary")
+                refresh_btn = gr.Button("🔄 Refresh Document Lists")
+            upload_output = gr.Textbox(label="Status")
+
+            def upload_and_refresh(file):
+                status = add_pdf_to_index(file)
+                d1, d2, ds, wc = refresh_doc_dropdowns()
+                return status, d1, d2, ds, wc
+
+            upload_btn.click(
+                fn=upload_and_refresh,
+                inputs=file_input,
+                outputs=[upload_output, doc1_input, doc2_input, doc_name_input, wc_doc_name]
+            )
+            refresh_btn.click(
+                fn=refresh_doc_dropdowns,
+                outputs=[doc1_input, doc2_input, doc_name_input, wc_doc_name]
+            )
+
+        # 10. ADVANCED ANALYTICS
+        with gr.TabItem("📈 Advanced Analytics"):
+            gr.Markdown("### Sentiment trends and entity frequencies")
+            analytics_input = gr.Textbox(label="Text to Analyze (Paste text or query)")
+            analytics_btn = gr.Button("Run Analytics")
+            analytics_output = gr.Markdown(label="Analytics Report")
+            analytics_btn.click(fn=advanced_analytics, inputs=analytics_input, outputs=analytics_output)
+
+        # 11. MULTI-MODEL COMPARISON
+        with gr.TabItem("🤖 Multi-Model Comparison"):
+            gr.Markdown("### Compare LLM Outputs (AI Grid vs OpenRouter)")
+            mm_query = gr.Textbox(label="Query")
+            with gr.Row():
+                m1_dropdown = gr.Dropdown(choices=MODELS, value="gpt-oss-120b", label="Primary Model")
+                m2_dropdown = gr.Dropdown(choices=MODELS, value="google/gemma-4-31B", label="Secondary Model")
+            mm_btn = gr.Button("Compare")
+            with gr.Row():
+                mm_out1 = gr.Textbox(label="Primary Output", lines=10)
+                mm_out2 = gr.Textbox(label="Secondary Output", lines=10)
+                
+            def compare_models(query, m1, m2):
+                ans1, _ = query_rag(query, model_id=m1)
+                ans2, _ = query_rag(query, model_id=m2)
+                return ans1, ans2
+                
+            mm_btn.click(fn=compare_models, inputs=[mm_query, m1_dropdown, m2_dropdown], outputs=[mm_out1, mm_out2])
+
+        # 12. ABOUT
+        with gr.TabItem("ℹ️ About"):
+            gr.Markdown(
+                """
+                ### About the Project
+                This Agentic RAG Platform is exclusively built to answer questions regarding the Palestinian cause based entirely on 15 official, verified documents.
+                
+                **Methodology:**
+                - Uses `LlamaIndex` for retrieval-augmented generation.
+                - Strict system prompts to prevent hallucination and reliance on external knowledge.
+                - Employs `intfloat/multilingual-e5-base` for dense multilingual embeddings.
+                - Supports AI Grid and OpenRouter models.
+                
+                **Features:**
+                - Multi-lingual Smart Chat with exact citations.
+                - Map, Timeline, Word Cloud, and Statistical visualization.
+                - Discourse & Sentiment Analysis.
+                """
+            )
+
+if __name__ == "__main__":
+    demo.launch(css=PALESTINE_CSS)

config.py

@@ -0,0 +1,29 @@
+import os
+
+# ── API Keys ──────────────────────────────────────────────
+AIGRID_API_KEY_GPT   = os.getenv("AIGRID_API_KEY_GPT", "sk-XaDrxkmJNvrp04SfkHT2ig")
+AIGRID_API_KEY_GEMMA = os.getenv("AIGRID_API_KEY_GEMMA", "sk-eQYZ67KgWjIMcPZb6SwwKg")
+OPENROUTER_API_KEY   = os.getenv("OPENROUTER_API_KEY", "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")
+
+# ── API Bases ─────────────────────────────────────────────
+AIGRID_API_BASE     = "http://app.ai-grid.io:4000/v1"
+OPENROUTER_API_BASE = "https://openrouter.ai/api/v1"
+
+# ── Model Settings ────────────────────────────────────────
+LLM_MODEL         = "gpt-oss-120b"
+# EMBEDDING_MODEL   = "intfloat/multilingual-e5-base"
+EMBEDDING_MODEL   = "sentence-transformers/all-MiniLM-L6-v2"
+LLM_TEMPERATURE   = 0.1                     # Low = less hallucination
+MAX_TOKENS        = 1024
+
+# ── Chunking Settings ─────────────────────────────────────
+CHUNK_SIZE        = 500
+CHUNK_OVERLAP     = 100
+
+# ── Retrieval Settings ────────────────────────────────────
+TOP_K             = 5
+SIMILARITY_CUTOFF = 0.3                     # Below this = "not found"
+
+# ── Paths ─────────────────────────────────────────────────
+DOCUMENTS_DIR     = "./documents"
+FAISS_INDEX_PATH  = "/data/faiss_index"

document_processor.py

@@ -0,0 +1,107 @@
+import os
+import fitz                          # PyMuPDF
+from langdetect import detect
+from config import DOCUMENTS_DIR, CHUNK_SIZE, CHUNK_OVERLAP
+
+
+# ── Detect Language ───────────────────────────────────────────────────────────
+def detect_language(text: str) -> str:
+    try:
+        lang = detect(text[:500])
+        return "ar" if lang == "ar" else "en"
+    except Exception:
+        return "en"
+
+
+# ── Load a Single PDF ─────────────────────────────────────────────────────────
+def load_pdf(pdf_path: str) -> list[dict]:
+    """
+    Returns a list of page dicts:
+    { text, page_number, source, language }
+    """
+    pages = []
+    doc_name = os.path.splitext(os.path.basename(pdf_path))[0]
+
+    try:
+        doc = fitz.open(pdf_path)
+        for i, page in enumerate(doc):
+            text = page.get_text().strip()
+            if not text:          # skip empty pages
+                continue
+            pages.append({
+                "text"       : text,
+                "page_number": i + 1,
+                "source"     : doc_name,
+                "language"   : detect_language(text),
+            })
+        doc.close()
+    except Exception as e:
+        print(f"[ERROR] Could not load {pdf_path}: {e}")
+
+    return pages
+
+
+# ── Chunk a List of Pages ─────────────────────────────────────────────────────
+def chunk_pages(pages: list[dict]) -> list[dict]:
+    """
+    Splits page text into overlapping chunks.
+    Each chunk keeps the source metadata.
+    """
+    chunks = []
+
+    for page in pages:
+        text       = page["text"]
+        words      = text.split()
+        start      = 0
+
+        while start < len(words):
+            end        = start + CHUNK_SIZE
+            chunk_text = " ".join(words[start:end])
+
+            chunks.append({
+                "text"       : chunk_text,
+                "page_number": page["page_number"],
+                "source"     : page["source"],
+                "language"   : page["language"],
+            })
+
+            start += CHUNK_SIZE - CHUNK_OVERLAP   # overlap
+
+    return chunks
+
+
+# ── Load ALL PDFs in the documents/ folder ───────────────────────────────────
+def load_all_documents() -> list[dict]:
+    all_chunks = []
+
+    if not os.path.exists(DOCUMENTS_DIR):
+        os.makedirs(DOCUMENTS_DIR)
+        print(f"[INFO] Created '{DOCUMENTS_DIR}' — add your PDFs there.")
+        return all_chunks
+
+    pdf_files = [
+        f for f in os.listdir(DOCUMENTS_DIR)
+        if f.lower().endswith(".pdf")
+    ]
+
+    if not pdf_files:
+        print(f"[WARN] No PDFs found in '{DOCUMENTS_DIR}'.")
+        return all_chunks
+
+    for pdf_file in pdf_files:
+        path   = os.path.join(DOCUMENTS_DIR, pdf_file)
+        pages  = load_pdf(path)
+        chunks = chunk_pages(pages)
+        all_chunks.extend(chunks)
+        print(f"[INFO] Loaded '{pdf_file}' → {len(chunks)} chunks")
+
+    print(f"[INFO] Total chunks: {len(all_chunks)}")
+    return all_chunks
+
+
+# ── Load a Single Uploaded PDF (for the Upload Tab) ──────────────────────────
+def load_uploaded_pdf(pdf_path: str) -> list[dict]:
+    pages  = load_pdf(pdf_path)
+    chunks = chunk_pages(pages)
+    print(f"[INFO] Uploaded PDF → {len(chunks)} chunks")
+    return chunks

documents/20241106-Gaza-Update-Report-OPT.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca826df600cf7a276355416eb5b0f5ec38b37f31f0173ce5d97124b81ecd263a
+size 1766016

documents/2024_04_20_UNRWA-final-technical_report.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:704638634b5678058fcec8bddab10a2b00f46cc120e099e0d6856a67e241f68e
+size 435406

documents/Humanitarian-Situation-Update-176-_-Gaza-Strip-_-United-Nations-Office-for-the-Coordination-of-Humanitarian-Affairs-occupied-Palestinian-territory.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c6086889da21082a49b31fdbbd162af2677b953c71672b13cf966c980e8f56
+size 307785

documents/Israel-Palestine-History-Timeline-2024-25-update.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55300e2f63012ef8cbac8f4af81e5bd61360dd6e55d2efac45d8e0f67bbac2d7
+size 1466687

documents/Khalidi-Rashid-Palestinian-Identity.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a544e1b992ca4fc4298fe7d4ee082eb3b6acb12873f6f2983851774539e3bc8
+size 10389182

documents/Palestinian-History-Calendar.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f760914b73636e9d1b609295b46963b1d73a9a35fb187d0b59349caaee36e026
+size 750558

documents/The Hundred Years’ War on Palestine.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af5e7b82c534dc20fedf4f2d4ab106927b710615a4dcbc5935e4e9b65695b738
+size 2734756

documents/ga_res_1941948.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55c90a3a1124692a6fe9e7cbbac41730b7eae9ed749fe5e61f767a650a6d6d41
+size 27740

documents/تقرير غزة الإنساني 2024.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df98cf9c8662ea534c9166200f7b26d4144958fd606605857fdf9b6a730579dc
+size 137750

documents/ذاكرة المكان.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c04d2e13d7dde437a00d2252f3f09e58ef3bd31d18124a901820422a89ee4f07
+size 718225

documents/شخصيات فلسطينية.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94de96d73ff18a7b774f185e1e11b12ec2802b006bfaad050f4a92ae1480bc3b
+size 189648

documents/فلسطين العربية.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1013f41ba06a1a6361e51ad36e494f685c07b5ec6ce91014b73947006c3a3f06
+size 2989231

documents/فلسطين.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e13ec4a75f910c406a9479996a4f75dd60f43a4310e63ab1f11aa2a780c07e2
+size 3516274

documents/كتاب-النخبة-1-.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b45c816283ff0382827e6f73d00a6e0d31d19b955ef8197d6cb3a42dc4321162
+size 7813088

documents/كتاب-النخبة-2-.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddd54a3b02b5fad12b4b138b9c69dc8c4cd040a9492408519df03fe9e35f0635
+size 12651217

extensions.py

@@ -0,0 +1,269 @@
+import os
+import subprocess
+import pandas as pd
+import folium
+from wordcloud import WordCloud
+import matplotlib.pyplot as plt
+from textblob import TextBlob
+from gtts import gTTS
+import speech_recognition as sr
+from deep_translator import GoogleTranslator
+from collections import Counter
+import re
+import json
+
+# We will import _index from rag_pipeline to get documents
+import rag_pipeline
+
+# ── Map Generation ──────────────────────────────────────────────
+LOCATIONS = [
+    {"name": "Jerusalem (Al-Quds)", "lat": 31.7683, "lon": 35.2137,
+     "query": "Jerusalem Al-Quds occupation history destruction",
+     "desc": "The capital of Palestine, central to its history, culture, and religious identity."},
+    {"name": "Gaza", "lat": 31.5017, "lon": 34.4668,
+     "query": "Gaza destruction casualties humanitarian crisis displaced",
+     "desc": "One of the oldest cities; subject of military operations and humanitarian siege."},
+    {"name": "Ramallah", "lat": 31.9038, "lon": 35.2034,
+     "query": "Ramallah West Bank Palestinian Authority",
+     "desc": "A major Palestinian cultural and political center in the West Bank."},
+    {"name": "Hebron (Al-Khalil)", "lat": 31.5326, "lon": 35.0998,
+     "query": "Hebron Al-Khalil settlements occupation",
+     "desc": "A historic city known for the Ibrahimi Mosque and traditional crafts."},
+    {"name": "Nablus", "lat": 32.2211, "lon": 35.2544,
+     "query": "Nablus West Bank raids settlements",
+     "desc": "Famous for its traditional soap, knafeh, and historic old city."},
+    {"name": "Haifa", "lat": 32.7940, "lon": 34.9896,
+     "query": "Haifa Nakba 1948 Palestinian expelled",
+     "desc": "A historic coastal city, largely depopulated during the 1948 Nakba."},
+    {"name": "Jaffa (Yafa)", "lat": 32.0504, "lon": 34.7522,
+     "query": "Jaffa Yafa Nakba 1948 destruction port expelled",
+     "desc": "Historically one of Palestine's most important port cities, depopulated in 1948."},
+    {"name": "Rafah", "lat": 31.2956, "lon": 34.2527,
+     "query": "Rafah crossing humanitarian aid evacuation bombardment",
+     "desc": "A border city in southern Gaza; key crossing for humanitarian aid."},
+    {"name": "Khan Yunis", "lat": 31.3436, "lon": 34.3061,
+     "query": "Khan Yunis destruction bombardment casualties",
+     "desc": "One of Gaza's largest cities, heavily affected by military operations."},
+    {"name": "Jenin", "lat": 32.4641, "lon": 35.2961,
+     "query": "Jenin refugee camp military operation incursion",
+     "desc": "Home to one of the West Bank's largest refugee camps."},
+]
+
+
+def _get_location_facts(query: str) -> str:
+    """Retrieve document excerpts relevant to a location. Returns formatted HTML."""
+    if rag_pipeline._retriever is None:
+        return ""
+    try:
+        nodes = rag_pipeline._retriever.retrieve(query)
+        if not nodes:
+            return ""
+        snippets = []
+        seen = set()
+        for node in nodes[:3]:
+            text   = node.node.get_content()[:280].strip().replace("\n", " ")
+            source = node.node.metadata.get("source", "")
+            page   = node.node.metadata.get("page_number", "?")
+            key    = (source, page)
+            if key in seen:
+                continue
+            seen.add(key)
+            src_label = (source[:45] + "...") if len(source) > 45 else source
+            snippets.append(
+                f'<blockquote style="font-size:11px;margin:4px 0;border-left:3px solid #c00;'
+                f'padding-left:6px;color:#222;">'
+                f'"{text}..."<br>'
+                f'<i style="color:#666;">&#8212; {src_label}, p.{page}</i>'
+                f'</blockquote>'
+            )
+        return "".join(snippets)
+    except Exception:
+        return ""
+
+
+def generate_map():
+    m = folium.Map(location=[31.5, 34.8], zoom_start=8, tiles="CartoDB positron")
+
+    for loc in LOCATIONS:
+        doc_facts = _get_location_facts(loc["query"])
+
+        popup_html = (
+            f'<div style="font-family:Arial,sans-serif;max-width:340px;direction:auto;">'
+            f'<h4 style="margin:0 0 6px;color:#1a1a1a;">{loc["name"]}</h4>'
+            f'<p style="font-size:12px;color:#333;margin:0 0 8px;">{loc["desc"]}</p>'
+        )
+        if doc_facts:
+            popup_html += (
+                f'<hr style="border:none;border-top:1px solid #ddd;margin:6px 0;">'
+                f'<p style="font-size:11px;font-weight:bold;color:#c00;margin:0 0 4px;">'
+                f'&#128196; From the Documents:</p>'
+                f'{doc_facts}'
+            )
+        popup_html += "</div>"
+
+        folium.Marker(
+            location=[loc["lat"], loc["lon"]],
+            popup=folium.Popup(popup_html, max_width=360),
+            tooltip=folium.Tooltip(loc["name"], sticky=True),
+            icon=folium.Icon(color="red", icon="info-sign"),
+        ).add_to(m)
+
+    map_path = "palestine_map.html"
+    m.save(map_path)
+    return map_path
+
+# ── Timeline Generation ──────────────────────────────────────────────
+def generate_timeline():
+    timeline_html = """
+    <div style="font-family: Arial, sans-serif; padding: 20px;">
+        <h3>Historical Timeline of the Palestinian Cause</h3>
+        <ul style="border-left: 2px solid #333; padding-left: 20px;">
+            <li style="margin-bottom: 10px;"><b>1917:</b> Balfour Declaration issued by the British government.</li>
+            <li style="margin-bottom: 10px;"><b>1947:</b> UN General Assembly adopts Resolution 181 (Partition Plan).</li>
+            <li style="margin-bottom: 10px;"><b>1948:</b> The Nakba (Catastrophe); hundreds of thousands of Palestinians displaced.</li>
+            <li style="margin-bottom: 10px;"><b>1967:</b> The Naksa (Setback); occupation of the West Bank, Gaza, and East Jerusalem.</li>
+            <li style="margin-bottom: 10px;"><b>1987:</b> The First Intifada begins.</li>
+            <li style="margin-bottom: 10px;"><b>1993:</b> Oslo Accords signed.</li>
+            <li style="margin-bottom: 10px;"><b>2000:</b> The Second Intifada begins.</li>
+            <li style="margin-bottom: 10px;"><b>Present:</b> Ongoing struggle for self-determination and human rights.</li>
+        </ul>
+    </div>
+    """
+    return timeline_html
+
+# ── Word Cloud Generation ──────────────────────────────────────────────
+def generate_wordcloud(doc_name="All"):
+    if rag_pipeline._index is None:
+        return None
+    
+    docstore = rag_pipeline._index.docstore
+    nodes = list(docstore.docs.values())
+    
+    text = ""
+    for node in nodes:
+        if doc_name is None or doc_name == "All" or node.metadata.get("source") == doc_name:
+            text += node.get_content() + " "
+            
+    if not text.strip():
+        # Fallback if no text
+        text = "Palestine History Culture Rights Peace Justice Freedom"
+        
+    wordcloud = WordCloud(width=800, height=400, background_color='white').generate(text)
+    plt.figure(figsize=(10, 5))
+    plt.imshow(wordcloud, interpolation='bilinear')
+    plt.axis('off')
+    img_path = "wordcloud.png"
+    plt.savefig(img_path, bbox_inches='tight')
+    plt.close()
+    return img_path
+
+# ── Statistics Generation ──────────────────────────────────────────────
+def get_statistics():
+    if rag_pipeline._index is None:
+        return pd.DataFrame(), pd.DataFrame()
+        
+    docstore = rag_pipeline._index.docstore
+    nodes = list(docstore.docs.values())
+    
+    data = []
+    for node in nodes:
+        source = node.metadata.get("source", "Unknown")
+        page = node.metadata.get("page_number", 0)
+        length = len(node.get_content())
+        data.append({"Source": source, "Page": page, "Length": length})
+        
+    df = pd.DataFrame(data)
+    
+    if df.empty:
+        return pd.DataFrame(), pd.DataFrame()
+        
+    stats_df = df.groupby('Source').agg(
+        Chunks=('Source', 'count'),
+        Total_Length=('Length', 'sum'),
+        Avg_Length=('Length', 'mean')
+    ).reset_index()
+    
+    return stats_df, df
+
+# ── Advanced Analytics ──────────────────────────────────────────────
+def advanced_analytics(text):
+    if not text or not text.strip():
+        return "No text provided for analysis."
+        
+    blob = TextBlob(text)
+    sentiment = blob.sentiment
+    sentiment_str = f"Polarity: {sentiment.polarity:.2f} (Negative < 0 < Positive), Subjectivity: {sentiment.subjectivity:.2f} (Objective < 0.5 < Subjective)"
+    
+    words = re.findall(r'\b[A-Z][a-z]+\b', text)
+    freq = Counter(words)
+    common_entities = freq.most_common(10)
+    
+    analytics_report = f"### Sentiment Analysis\n{sentiment_str}\n\n"
+    analytics_report += "### Frequent Capitalized Entities (Heuristic)\n"
+    for ent, count in common_entities:
+        analytics_report += f"- **{ent}**: {count}\n"
+        
+    return analytics_report
+
+# ── Audio Features ──────────────────────────────────────────────
+def text_to_speech(text, lang='en'):
+    try:
+        tts = gTTS(text=text, lang=lang)
+        output_path = "output_audio.mp3"
+        tts.save(output_path)
+        return output_path
+    except Exception as e:
+        print(f"TTS Error: {e}")
+        return None
+
+def speech_to_text(audio_path):
+    if not audio_path:
+        return ""
+        
+    wav_path = audio_path
+    if not audio_path.lower().endswith(".wav"):
+        wav_path = "temp_stt.wav"
+        try:
+            subprocess.run(["ffmpeg", "-y", "-i", audio_path, wav_path], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+        except Exception as e:
+            error_msg = f"[Voice Input Error: Audio conversion failed: {str(e)}]"
+            print(error_msg)
+            return error_msg
+            
+    r = sr.Recognizer()
+    try:
+        with sr.AudioFile(wav_path) as source:
+            audio_data = r.record(source)
+            text = r.recognize_google(audio_data)
+            return text
+    except Exception as e:
+        error_msg = f"[Voice Input Error: {str(e)}]"
+        print(error_msg)
+        return error_msg
+
+# ── Translation ──────────────────────────────────────────────
+def translate_text(text, target_lang='en'):
+    try:
+        translator = GoogleTranslator(source='auto', target=target_lang)
+        return translator.translate(text)
+    except Exception as e:
+        print(f"Translation Error: {e}")
+        return text
+
+# ── Export Chat ──────────────────────────────────────────────
+def export_chat_history(history):
+    if not history:
+        return None
+    
+    # Convert objects to dicts if necessary for JSON serialization
+    cleaned_history = []
+    for msg in history:
+        if isinstance(msg, dict):
+            cleaned_history.append(msg)
+        else:
+            cleaned_history.append({"role": getattr(msg, "role", "unknown"), "content": getattr(msg, "content", "")})
+            
+    file_path = "chat_history.json"
+    with open(file_path, "w", encoding="utf-8") as f:
+        json.dump(cleaned_history, f, ensure_ascii=False, indent=4)
+    return file_path

rag_pipeline.py

@@ -0,0 +1,371 @@
+import os
+import pickle
+from langdetect import detect
+
+from llama_index.core                        import VectorStoreIndex, Document, Settings
+from llama_index.core.retrievers             import VectorIndexRetriever
+from llama_index.embeddings.huggingface      import HuggingFaceEmbedding
+from llama_index.llms.openai_like            import OpenAILike
+from llama_index.core.node_parser            import SimpleNodeParser
+
+from document_processor import load_all_documents, load_uploaded_pdf
+from config import (
+    AIGRID_API_KEY_GPT, AIGRID_API_KEY_GEMMA, AIGRID_API_BASE, 
+    OPENROUTER_API_KEY, OPENROUTER_API_BASE,
+    LLM_MODEL, EMBEDDING_MODEL,
+    LLM_TEMPERATURE, MAX_TOKENS, TOP_K, SIMILARITY_CUTOFF,
+    FAISS_INDEX_PATH,
+)
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Global state
+# ─────────────────────────────────────────────────────────────────────────────
+_index     = None      # LlamaIndex VectorStoreIndex
+_retriever = None      # Retriever object
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# LLM Factory
+# ─────────────────────────────────────────────────────────────────────────────
+def get_llm(model_id: str):
+    if model_id == "gpt-oss-120b":
+        return OpenAILike(
+            model       = model_id,
+            api_key     = AIGRID_API_KEY_GPT,
+            api_base    = AIGRID_API_BASE,
+            temperature = LLM_TEMPERATURE,
+            max_tokens  = MAX_TOKENS,
+            is_chat_model=True,
+        )
+    elif model_id == "google/gemma-4-31B":
+        return OpenAILike(
+            model       = model_id,
+            api_key     = AIGRID_API_KEY_GEMMA,
+            api_base    = AIGRID_API_BASE,
+            temperature = LLM_TEMPERATURE,
+            max_tokens  = MAX_TOKENS,
+            is_chat_model=True,
+        )
+    else:
+        # Assume it's an OpenRouter model
+        return OpenAILike(
+            model       = model_id,
+            api_key     = OPENROUTER_API_KEY,
+            api_base    = OPENROUTER_API_BASE,
+            temperature = LLM_TEMPERATURE,
+            max_tokens  = MAX_TOKENS,
+            is_chat_model=True,
+        )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Initialise models
+# ─────────────────────────────────────────────────────────────────────────────
+def _init_models():
+    embed_model = HuggingFaceEmbedding(model_name=EMBEDDING_MODEL)
+    
+    # We use a default LLM for any global index operations if necessary
+    default_llm = get_llm(LLM_MODEL)
+
+    # Apply globally to LlamaIndex
+    Settings.embed_model = embed_model
+    Settings.llm         = default_llm
+    Settings.chunk_size  = 512       # internal safety
+
+    print("[INFO] Models initialised.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Build index from chunk dicts
+# ─────────────────────────────────────────────────────────────────────────────
+def _build_index(chunks: list[dict]) -> VectorStoreIndex:
+    documents = []
+    for chunk in chunks:
+        doc = Document(
+            text     = chunk["text"],
+            metadata = {
+                "source"     : chunk["source"],
+                "page_number": chunk["page_number"],
+                "language"   : chunk["language"],
+            },
+        )
+        documents.append(doc)
+
+    index = VectorStoreIndex.from_documents(
+        documents,
+        show_progress=True,
+    )
+    return index
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: initialise everything at startup
+# ─────���───────────────────────────────────────────────────────────────────────
+def initialise_pipeline():
+    global _index, _retriever
+
+    _init_models()
+
+    chunks = load_all_documents()
+
+    if not chunks:
+        print("[WARN] No documents loaded — index will be empty.")
+        chunks = [{"text": "placeholder", "page_number": 1,
+                   "source": "none", "language": "en"}]
+
+    _index     = _build_index(chunks)
+    _retriever = VectorIndexRetriever(index=_index, similarity_top_k=TOP_K)
+
+    print("[INFO] Pipeline ready.")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: add a new uploaded PDF to the existing index
+# ─────────────────────────────────────────────────────────────────────────────
+def add_pdf_to_index(pdf_path: str) -> str:
+    global _index, _retriever
+
+    if _index is None:
+        return "❌ Pipeline not initialised yet."
+
+    new_chunks = load_uploaded_pdf(pdf_path)
+
+    if not new_chunks:
+        return "❌ Could not extract text from this PDF."
+
+    for chunk in new_chunks:
+        doc = Document(
+            text     = chunk["text"],
+            metadata = {
+                "source"     : chunk["source"],
+                "page_number": chunk["page_number"],
+                "language"   : chunk["language"],
+            },
+        )
+        _index.insert(doc)
+
+    _retriever = VectorIndexRetriever(index=_index, similarity_top_k=TOP_K)
+
+    return (
+        f"✅ PDF indexed successfully!\n"
+        f"📄 {len(new_chunks)} chunks added.\n"
+        f"🔍 Ready to query!"
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Internal: retrieve relevant nodes
+# ─────────────────────────────────────────────────────────────────────────────
+def _retrieve(query: str):
+    if _retriever is None:
+        return []
+    nodes = _retriever.retrieve(query)
+    return nodes
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Internal: detect query language
+# ─────────────────────────────────────────────────────────────────────────────
+def _detect_lang(text: str) -> str:
+    try:
+        lang = detect(text[:500])
+        return "ar" if lang == "ar" else "en"
+    except Exception:
+        return "en"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Internal: build answer prompt
+# ─────────────────────────────────────────────────────────────────────────────
+def _build_prompt(query: str, nodes, lang: str) -> str:
+    context_parts = []
+    for i, node in enumerate(nodes):
+        meta   = node.metadata
+        source = meta.get("source", "Unknown")
+        page   = meta.get("page_number", "?")
+        text   = node.get_content()
+        context_parts.append(f"[{i+1}] Source: {source} | Page: {page}\n{text}")
+
+    context = "\n\n".join(context_parts)
+
+    if lang == "ar":
+        instruction = (
+            "أنت مساعد متخصص في الوثائق الفلسطينية.\n"
+            "القواعد الصارمة:\n"
+            "1. أجب فقط بناءً على الوثائق المسترجعة\n"
+            "2. اذكر دائماً: اسم الوثيقة + رقم الصفحة\n"
+            "3. إذا لم تجد الإجابة، قل: 'لم يتم العثور على هذه المعلومات في الوثائق المتاحة'\n"
+            "4. لا تستخدم معرفتك الخارجية أبداً\n"
+        )
+    else:
+        instruction = (
+            "You are a document-based assistant specialising in Palestinian documents.\n"
+            "STRICT RULES:\n"
+            "1. Answer ONLY from the retrieved document chunks below\n"
+            "2. ALWAYS cite: Document Title + Page Number\n"
+            "3. If the answer is NOT in the documents, say exactly: "
+            "'This information was not found in the provided documents.'\n"
+            "4. NEVER use external knowledge\n"
+            "5. Respond in the SAME language as the question\n"
+        )
+
+    prompt = (
+        f"{instruction}\n"
+        f"Retrieved Context:\n{context}\n\n"
+        f"Question: {query}\n"
+        f"Answer:"
+    )
+    return prompt
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: main query function
+# ─────────────────────────────────────────────────────────────────────────────
+def query_rag(question: str, model_id: str = "gpt-oss-120b") -> tuple[str, list[dict]]:
+    """
+    Returns (answer_text, list_of_source_dicts)
+    """
+    if not question.strip():
+        return "Please enter a question.", []
+
+    lang  = _detect_lang(question)
+    nodes = _retrieve(question)
+
+    # ── Anti-hallucination gate ───────────────────────────────────────────────
+    if not nodes:
+        if lang == "ar":
+            return "لم يتم العثور على هذه المعلومات في الوثائق المتاحة.", []
+        return "This information was not found in the provided documents.", []
+
+    # Check similarity scores
+    top_score = max((n.score for n in nodes if n.score is not None), default=0)
+    if top_score < SIMILARITY_CUTOFF:
+        if lang == "ar":
+            return "لم يتم العثور على هذه المعلومات في الوثائق المتاحة.", []
+        return "This information was not found in the provided documents.", []
+
+    # ── Build prompt & call LLM ───────────────────────────────────────────────
+    prompt = _build_prompt(question, nodes, lang)
+    llm = get_llm(model_id)
+
+    try:
+        response = llm.complete(prompt)
+        answer   = str(response)
+    except Exception as e:
+        return f"❌ LLM error: {e}", []
+
+    # ── Build sources list ────────────────────────────────────────────────────
+    sources = []
+    seen    = set()
+    for node in nodes:
+        meta = node.metadata
+        key  = (meta.get("source", ""), meta.get("page_number", ""))
+        if key not in seen:
+            seen.add(key)
+            sources.append({
+                "source"  : meta.get("source", "Unknown"),
+                "page"    : meta.get("page_number", "?"),
+                "language": meta.get("language", "en"),
+                "score"   : round(node.score, 3) if node.score else 0,
+                "snippet" : node.get_content()[:200] + "...",
+            })
+
+    return answer, sources
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: summarise a specific document
+# ─────────────────────────────────────────────────────────────────────────────
+def summarise_document(doc_name: str, model_id: str = "gpt-oss-120b") -> str:
+    if _retriever is None:
+        return "Pipeline not ready."
+
+    query = f"summarize the document {doc_name}"
+    nodes = _retriever.retrieve(query)
+
+    if not nodes:
+        return f"No content found for '{doc_name}'."
+
+    context = "\n\n".join(
+        [n.get_content()[:300] for n in nodes[:5]]
+    )
+
+    prompt = (
+        f"Summarise the following excerpts from the document '{doc_name}' "
+        f"in 5–7 bullet points. Be concise and factual.\n\n"
+        f"Content:\n{context}\n\nSummary:"
+    )
+
+    llm = get_llm(model_id)
+    try:
+        response = llm.complete(prompt)
+        return str(response)
+    except Exception as e:
+        return f"❌ Error: {e}"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: compare two documents
+# ────────────────────────────────────────────────────────��────────────────────
+def compare_documents(doc1: str, doc2: str, aspect: str = "main themes", model_id: str = "gpt-oss-120b") -> str:
+    if _retriever is None:
+        return "Pipeline not ready."
+
+    nodes1 = _retriever.retrieve(f"content of {doc1}")
+    nodes2 = _retriever.retrieve(f"content of {doc2}")
+
+    ctx1 = "\n".join([n.get_content()[:200] for n in nodes1[:3]])
+    ctx2 = "\n".join([n.get_content()[:200] for n in nodes2[:3]])
+
+    prompt = (
+        f"Compare these two documents regarding '{aspect}'.\n\n"
+        f"Document 1 — {doc1}:\n{ctx1}\n\n"
+        f"Document 2 — {doc2}:\n{ctx2}\n\n"
+        f"Provide a structured comparison with:\n"
+        f"- Similarities\n"
+        f"- Differences\n"
+        f"- Key Takeaways\n"
+        f"Comparison:"
+    )
+
+    llm = get_llm(model_id)
+    try:
+        response = llm.complete(prompt)
+        return str(response)
+    except Exception as e:
+        return f"❌ Error: {e}"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Public: discourse / bias analysis
+# ─────────────────────────────────────────────────────────────────────────────
+def analyse_discourse(query: str, model_id: str = "gpt-oss-120b") -> str:
+    if _retriever is None:
+        return "Pipeline not ready."
+
+    nodes = _retriever.retrieve(query)
+
+    if not nodes:
+        return "No relevant content found."
+
+    context = "\n\n".join([n.get_content()[:300] for n in nodes[:4]])
+
+    prompt = (
+        f"Perform a discourse analysis on the following text from Palestinian documents.\n"
+        f"Identify:\n"
+        f"1. 🏷️ Key Terminology & Framing\n"
+        f"2. ⚖️ Bias indicators (if any)\n"
+        f"3. 🔁 Repeated Narratives\n"
+        f"4. 💬 Tone & Language\n"
+        f"5. 🎯 Apparent Purpose/Agenda\n\n"
+        f"Topic/Query: {query}\n\n"
+        f"Text:\n{context}\n\n"
+        f"Analysis:"
+    )
+
+    llm = get_llm(model_id)
+    try:
+        response = llm.complete(prompt)
+        return str(response)
+    except Exception as e:
+        return f"❌ Error: {e}"

requirements.txt

@@ -0,0 +1,35 @@
+# Core
+gradio>=4.0.0
+python-dotenv
+
+# LlamaIndex
+llama-index-core
+llama-index-llms-openai-like
+llama-index-embeddings-huggingface
+
+# PDF Processing
+pymupdf
+pdfplumber
+
+# Language Detection & Translation
+langdetect
+deep-translator
+
+# Visualisation
+plotly
+matplotlib
+wordcloud
+folium
+pandas
+Pillow
+numpy
+
+# Analytics & Voice
+textblob
+gTTS
+SpeechRecognition
+
+# ML / Embeddings
+sentence-transformers
+torch
+transformers