Upload 5 files

.gitattributes

@@ -1,44 +1,44 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-faiss_index_unmad_magz/index.faiss filter=lfs diff=lfs merge=lfs -text
-images/354043071_653375850167696_1862718270923061923_n-removebg-preview.png filter=lfs diff=lfs merge=lfs -text
-images/unmad_pic.png filter=lfs diff=lfs merge=lfs -text
-images/c1.png filter=lfs diff=lfs merge=lfs -text
-lance_unmad_db/unmad_documents.lance/data/1ed6346a-63e5-488b-8266-93cb400a38fa.lance filter=lfs diff=lfs merge=lfs -text
-lance_unmad_db/unmad_documents.lance/data/fb6df230-9aa6-455d-b5a8-eb3fe5acda98.lance filter=lfs diff=lfs merge=lfs -text
-lance_unmad_db/unmad_documents.lance/data/78e3ec7c-8251-40e9-b828-8f3610f16dc6.lance filter=lfs diff=lfs merge=lfs -text
-lance_unmad_db/unmad_documents.lance/data/974f34e7-a8d6-4bb7-b0f6-53cd0f4a684b.lance filter=lfs diff=lfs merge=lfs -text
-lance_unmad_db/unmad_documents.lance/data/ad577893-15ee-4592-bc76-65bbd21939d4.lance filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index_unmad_magz/index.faiss filter=lfs diff=lfs merge=lfs -text
+images/354043071_653375850167696_1862718270923061923_n-removebg-preview.png filter=lfs diff=lfs merge=lfs -text
+images/unmad_pic.png filter=lfs diff=lfs merge=lfs -text
+images/c1.png filter=lfs diff=lfs merge=lfs -text
+lance_unmad_db/unmad_documents.lance/data/1ed6346a-63e5-488b-8266-93cb400a38fa.lance filter=lfs diff=lfs merge=lfs -text
+lance_unmad_db/unmad_documents.lance/data/fb6df230-9aa6-455d-b5a8-eb3fe5acda98.lance filter=lfs diff=lfs merge=lfs -text
+lance_unmad_db/unmad_documents.lance/data/78e3ec7c-8251-40e9-b828-8f3610f16dc6.lance filter=lfs diff=lfs merge=lfs -text
+lance_unmad_db/unmad_documents.lance/data/974f34e7-a8d6-4bb7-b0f6-53cd0f4a684b.lance filter=lfs diff=lfs merge=lfs -text
+lance_unmad_db/unmad_documents.lance/data/ad577893-15ee-4592-bc76-65bbd21939d4.lance filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,4 @@
+.env
+__pycache__/
+*.pyc
+chat_logs/

app.py

@@ -1,382 +1,503 @@
-import os
-import re
-import json
-import warnings
-from typing import List, Dict, Any, Optional
-import lancedb
-import gradio as gr
-import numpy as np
-import pandas as pd
-from datetime import datetime
-from dotenv import load_dotenv
-from openai import OpenAI
-from sklearn.metrics.pairwise import cosine_similarity
-
-# Load environment variables
-load_dotenv()
-OPENAI_API_KEY = os.getenv("OPENAI_API_KEY_Project")
-if not OPENAI_API_KEY:
-    raise ValueError("Missing OPENAI_API_KEY. Please set it in your environment variables.")
-
-# Suppress warnings
-warnings.filterwarnings("ignore")
-
-class LanceDBRAG:
-    def __init__(self, 
-                 db_path: str = "lance_unmad_db",
-                 table_name: str = "unmad_documents"):
-        """Initialize LanceDB RAG System"""
-        self.db_path = db_path
-        self.table_name = table_name
-        
-        # Initialize OpenAI client
-        self.client = OpenAI(api_key=OPENAI_API_KEY)
-        
-        # Connect to LanceDB
-        try:
-            self.db = lancedb.connect(self.db_path)
-            self.table = self.db.open_table(self.table_name)
-            print(f"Connected to LanceDB: {self.db_path}/{self.table_name}")
-        except Exception as e:
-            raise ConnectionError(f"Failed to connect to LanceDB: {e}")
-
-    def get_embedding(self, text: str) -> List[float]:
-        """Get OpenAI embedding for query text"""
-        try:
-            response = self.client.embeddings.create(
-                model="text-embedding-3-small",
-                input=text
-            )
-            return response.data[0].embedding
-        except Exception as e:
-            print(f"Error getting embedding: {e}")
-            return None
-
-    def search_similar_content(self, query: str, limit: int = 10) -> pd.DataFrame:
-        """Search for similar content in the database"""
-        print(f"Searching: '{query}'")
-        
-        # Get query embedding
-        query_embedding = self.get_embedding(query)
-        if not query_embedding:
-            return pd.DataFrame()
-        
-        # Perform vector search
-        try:
-            search_query = self.table.search(query_embedding).limit(limit)
-            results = search_query.to_pandas()
-            
-            if not results.empty:
-                print(f"Found {len(results)} relevant results")
-            else:
-                print("No results found")
-            
-            return results
-            
-        except Exception as e:
-            print(f"Search error: {e}")
-            return pd.DataFrame()
-
-# Initialize global RAG instance
-rag_system = LanceDBRAG()
-
-def maximal_marginal_relevance_search(query, rag_instance, k=10, lambda_param=0.6, top_k=3):
-    """
-    Implement Maximal Marginal Relevance (MMR) for diverse document retrieval using LanceDB.
-    
-    Args:
-        query: Search query string
-        rag_instance: LanceDB RAG instance
-        k: Number of candidate documents to consider
-        lambda_param: Trade-off between relevance and diversity (0-1)
-        top_k: Number of final documents to return
-    
-    Returns:
-        List of selected documents with MMR ranking
-    """
-    # Get initial candidate documents using LanceDB search
-    search_results = rag_instance.search_similar_content(query, limit=k)
-    
-    if search_results.empty:
-        return []
-    
-    # Convert to document-like objects for compatibility
-    docs = []
-    for _, row in search_results.iterrows():
-        doc_obj = {
-            'page_content': row['text'],
-            'metadata': {
-                'source': row.get('magazine_name', 'Unknown Source'),
-                'page': row.get('page_number', 'Unknown Page'),
-                'chunk': row.get('chunk_id', 0)
-            },
-            'score': row['_distance']
-        }
-        docs.append(doc_obj)
-    
-    # Apply MMR selection if we have enough documents
-    if len(docs) <= top_k:
-        return docs[:top_k]
-    
-    # MMR Selection Algorithm
-    selected_docs = []
-    remaining_indices = list(range(len(docs)))
-    
-    for _ in range(min(top_k, len(docs))):
-        if not remaining_indices:
-            break
-            
-        mmr_scores = []
-        
-        for i in remaining_indices:
-            # Calculate relevance score (inverse of distance)
-            relevance = 1 / (1 + docs[i]['score'])
-            
-            # Calculate diversity score (max similarity to already selected docs)
-            if selected_docs:
-                max_similarity = 0
-                for selected_doc in selected_docs:
-                    # Simple text-based similarity for diversity
-                    text1 = docs[i]['page_content']
-                    text2 = selected_doc['page_content']
-                    
-                    # Calculate simple Jaccard similarity
-                    words1 = set(text1.split())
-                    words2 = set(text2.split())
-                    if words1 and words2:
-                        similarity = len(words1.intersection(words2)) / len(words1.union(words2))
-                        max_similarity = max(max_similarity, similarity)
-                
-                diversity = max_similarity
-            else:
-                diversity = 0
-            
-            # Calculate MMR score
-            mmr_score = lambda_param * relevance - (1 - lambda_param) * diversity
-            mmr_scores.append((mmr_score, i))
-        
-        # Select document with highest MMR score
-        if mmr_scores:
-            best_score, best_idx = max(mmr_scores, key=lambda x: x[0])
-            selected_docs.append(docs[best_idx])
-            remaining_indices.remove(best_idx)
-    
-    return selected_docs
-
-def clean_bangla_content(text):
-    """
-    Clean the retrieved content to remove English watermarks, scan text, and tags.
-    Keeps the Bengali text intact instead of deleting the whole line.
-    """
-    if not text:
-        return ""
-
-    # 1. First, strip out the specific tags entirely using NO backslashes
-    text = re.sub("[[]source: *[0-9]+[]]", "", text, flags=re.IGNORECASE)
-    
-    # Common English watermarks and scan text to remove
-    english_patterns = [
-        r'scanned by \w+',
-        r'found in \w+',
-        r'www\.\w+\.\w+',
-        r'http[s]?://[^\s]+',
-        r'\.pdf',
-        r'\.com',
-        r'\.org',
-        r'\.net',
-        r'banglapdf',
-        r'sadaqpdf',
-        r'pdf scanner',
-        r'scan by',
-        r'converted by',
-        r'page \d+',
-        r'source:', 
-        r'reference:',
-        r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+',  # emails
-    ]
-    
-    lines = text.split('\n')
-    cleaned_lines = []
-    
-    for line in lines:
-        # 2. Replace matched English watermark patterns with an empty string
-        for pattern in english_patterns:
-            line = re.sub(pattern, '', line, flags=re.IGNORECASE)
-            
-        line = line.strip()
-        if not line:
-            continue
-            
-        # 3. Check if there's actual Bengali content left
-        bengali_chars = len(re.findall(r'[\u0980-\u09FF]', line))
-        
-        # Only keep the line if it has Bengali characters
-        if bengali_chars > 0:
-            cleaned_lines.append(line)
-    
-    return '\n'.join(cleaned_lines)
-
-# Enhanced Satirical QA function with MMR and content cleaning
-def custom_unmad_satirical_bot(message, history, top_k=3, lambda_param=0.6):
-    """
-    Enhanced satirical bot using MMR for diverse and relevant content retrieval.
-    
-    Args:
-        message: User query
-        history: Chat history
-        top_k: Number of documents to retrieve
-        lambda_param: MMR trade-off (higher favors exact relevance over diversity)
-    """
-    # Use MMR search with LanceDB
-    docs = maximal_marginal_relevance_search(
-        query=message, 
-        rag_instance=rag_system, 
-        k=15,  # Consider more candidates for better diversity
-        lambda_param=lambda_param, 
-        top_k=top_k
-    )
-    
-    # Extract context from MMR-selected documents
-    if docs:
-        # Clean each document's content before joining
-        cleaned_contexts = []
-        for doc in docs:
-            cleaned_content = clean_bangla_content(doc['page_content'])
-            if cleaned_content.strip():  # Only add if there's meaningful Bengali content
-                cleaned_contexts.append(cleaned_content)
-        
-        if cleaned_contexts:
-            top_contexts = "\n\n---\n\n".join(cleaned_contexts)
-        else:
-            top_contexts = "No relevant information were found"
-        
-        # Add metadata about source diversity (optional)
-        source_info = []
-        for i, doc in enumerate(docs, 1):
-            source = doc['metadata'].get('source', 'Unknown source')
-            page = doc['metadata'].get('page', 'Unknown page')
-            # Clean source info too
-            if not re.search(r'[a-zA-Z]', str(source)):  # Only if source doesn't contain English
-                source_info.append(f"[{i}] {source} - {page}")
-        
-        source_context = "Source: " + " | ".join(source_info[:3]) if source_info else ""
-    else:
-        top_contexts = "No relevant information were found"
-        source_context = ""
-
-# Prepare system prompt
-    system_prompt = """
-তুমি বাংলাদেশের জনপ্রিয় স্যাটায়ার ম্যাগাজিন 'উন্মাদ'-এর একজন অত্যন্ত বুদ্ধিদীপ্ত, রসিক এবং কিছুটা 'তেড়া' (cynical) স্বভাবের লেখক। তোমার কাজ হলো ব্যবহারকারীর প্রশ্নের এমন উত্তর দেওয়া, যা পড়ে মনে হবে ঢাকার কোনো টং দোকানে বসে আড্ডা দেওয়ার সময় কেউ চরম স্যাটায়ার করছে। উত্তরগুলো হতে হবে লোকাল, চটপটে, এবং স্মার্ট।
-
-**এই নির্দেশনাগুলো কঠোরভাবে মেনে চলবে - কোন ব্যতিক্রম নেই:**
-
-১। **লোকাল ও খাঁটি দেশি রসিকতা:** ভিনগ্রহ, পিরামিড, বা রূপকথার মতো আজগুবি বা ভিনদেশি উদাহরণ একদম টানবে না। খাঁটি বাংলাদেশি মেটাফোর ব্যবহার করবে (যেমন: টংয়ের চা, ঢাকার জ্যাম, লোডশেডিং, কারওয়ান বাজারের কাদা, লোকাল বাসের হেলপার, ইত্যাদি)। কথাবার্তা হবে একদম ন্যাচারাল, যেন মুখোমুখি বসে আড্ডা দিচ্ছ।
-২। **প্রসঙ্গের ওপর সর্বোচ্চ জোর:** আমি তোমাকে যে 'প্রসঙ্গ' (Context) দেব, তোমার উত্তরের মূল ভিত্তি হবে সেটাই। প্রসঙ্গের বাইরের কোনো মনগড়া গল্প বানাবে না, বরং প্রসঙ্গের তথ্যগুলোকে বাঁকাভাবে উপস্থাপন করে খোঁচা দেবে।
-৩। **অচেনা নাম নিয়ে দেশি স্টাইলে কটাক্ষ:** ব্যবহারকারী যদি এমন কোনো নাম বা বিষয় নিয়ে প্রশ্ন করে যা তোমার 'প্রসঙ্গে' নেই (যেমন ইশরাকের সাথে সাথে 'নগর ড্যাডি'), তাহলে এমনভাবে খোঁচা দেবে যেন সে গুলিস্তানের মোড়ে দাঁড়িয়ে গাঁজাখুরি গল্প করছে। (যেমন বলতে পারো: "নগর ড্যাডি ইশরাক আবার কে ভাই? গুলিস্তানের নতুন কোনো চাঁদাবাজ নাকি? নাকি চাঁদা চাইতে গিয়ে নিজেই চাঁদা হয়ে গেছে?")
-৪। **খোঁচা দেওয়ার ধরন:** প্রসঙ্গের তথ্য ব্যবহার করে চরম স্যাটায়ার করবে। (যেমন: পিনাকী বা ইলিয়াসের কথা এলে বলবে তারা ফ্রান্স বা আমেরিকায় বসে 'ল্যাপটপ বিপ্লব' করছে; প্রথম আলোর কথা এলে বলবে আগুনে পোড়া ফার্নিচার দিয়ে আর্ট গ্যালারি বানানোর কথা)।
-৫। কোন ইমোজি (EMOJI), ইংরেজি শব্দ, ইংরেজি অক্ষর, ইংরেজি সংখ্যা বা স্ক্যান ওয়াটারমার্ক ব্যবহার করবে না। সম্পূর্ণ বাংলায় স্মার্ট, রসবোধ সম্পন্ন এবং 'উন্মাদ' স্টাইলে তীক্ষ্ণ উত্তর লেখো।
-"""
-
-    user_prompt = f"""
-প্রসঙ্গ (বিভিন্ন উৎস থেকে সংগৃহীত): 
-{top_contexts} 
-
-প্রশ্ন: {message} 
-
-নির্দেশনা: উপরের প্রসঙ্গ থেকে শুধুমাত্র বাংলা ভাষার বিষয়বস্তু ব্যবহার করে উন্মাদ ম্যাগাজিনের দেশি স্টাইলে উত্তর দাও। কোন ইমোজি বা ইংরেজি শব্দ দেবে না। উত্তরের ভাষা হবে চটপটে, ব্যঙ্গাত্মক এবং খাঁটি বাংলাদেশি। 
-
-""" 
-
-    # Generate response using OpenAI
-    try:
-        response = rag_system.client.chat.completions.create(
-            model="gpt-4o",
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": user_prompt}
-            ],
-            temperature=0.7,
-            max_tokens=700
-        )
-        
-        ai_response = response.choices[0].message.content
-        history.append((message, ai_response))
-        return "", history
-        
-    except Exception as e:
-        error_response = f"উত্তর তৈরিতে সমস্যা হয়েছে। আবার চেষ্টা করুন।"
-        history.append((message, error_response))
-        return "", history
-
-# Enhanced Gradio UI with Polished Layout and Fixed State Management
-with gr.Blocks(theme=gr.themes.Soft(), css=".gradio-container {padding-top: 40px;}") as demo:
-    
-    with gr.Column(elem_classes="text-center"):
-        gr.Markdown("# USB: Unmad Satirical Bot")
-        gr.Markdown("### *A Satirical Bot that impersonates the satirical character UNMAD*") 
-        
-        try:
-            # Upgraded Image block for a full-width banner look
-            gr.Image(
-                "images/c1.png", 
-                height=250,                 # Adjust this number to make the banner taller or shorter
-                show_label=False, 
-                container=False,            # Removes the bulky white background box
-                interactive=False,          # Makes it a static image (removes upload/edit hover buttons)
-                show_download_button=False, # Hides the download icon
-                show_fullscreen_button=False # Hides the fullscreen icon
-            )
-        except:
-            gr.Markdown("*[UNMAD Logo]*")
-
-    # Upgraded Chatbot component with avatars and better height
-    chatbot = gr.Chatbot(
-        height=500,
-        type="tuples",
-        avatar_images=(None, "images/c1.png"), # Left: User (Default), Right: Unmad (Your logo)
-        show_label=False
-    )
-
-    with gr.Row():
-        msg = gr.Textbox(
-            placeholder="কি চলে আপনার মনে? বলেন শুনি...", 
-            scale=8, 
-            show_label=False,
-            container=False # Removes the bulky box around the input
-        )
-        send = gr.Button("Send (↲)", variant="primary", scale=1)
-
-    clear = gr.Button("Clear Chat", variant="secondary")
-    
-    # State to keep track of the conversation
-    state = gr.State([])
-
-    def chat_with_fixed_mmr(message, history):
-        if not message.strip():
-            return "", history, history
-            
-        # Call your core function with increased top_k and higher lambda_param for strict relevance
-        _, updated_history = custom_unmad_satirical_bot(message, history, top_k=4, lambda_param=0.85)
-        
-        # Return: 1. Empty Textbox, 2. Visual Chatbot update, 3. Internal State update
-        return "", updated_history, updated_history
-
-    # Event Listeners with Loading States
-    # Disable the input box while generating to prevent spam
-    msg.submit(
-        chat_with_fixed_mmr, 
-        inputs=[msg, state], 
-        outputs=[msg, chatbot, state]
-    )
-    
-    send.click(
-        chat_with_fixed_mmr, 
-        inputs=[msg, state], 
-        outputs=[msg, chatbot, state]
-    )
-    
-    # Properly clear both the visual chatbot and the internal state
-    clear.click(lambda: ([], []), None, [chatbot, state], queue=False)
-
-if __name__ == "__main__":
+import os
+import re
+import csv
+import json
+import uuid
+import time
+import threading
+import warnings
+from typing import List, Dict, Any, Optional
+import lancedb
+import gradio as gr
+import numpy as np
+import pandas as pd
+from datetime import datetime, timezone
+from dotenv import load_dotenv
+from openai import OpenAI
+from sklearn.metrics.pairwise import cosine_similarity
+from huggingface_hub import HfApi, CommitScheduler
+
+# Load environment variables
+load_dotenv()
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY_Project")
+if not OPENAI_API_KEY:
+    raise ValueError("Missing OPENAI_API_KEY. Please set it in your environment variables.")
+
+# Suppress warnings
+warnings.filterwarnings("ignore")
+
+# ---------------------------------------------------------------------------
+# Chat-history persistence to a (private) Hugging Face Dataset repo
+# ---------------------------------------------------------------------------
+# Required Space secrets (set in Space -> Settings -> Variables and secrets):
+#   HF_TOKEN          -> a HF access token with WRITE permission
+#   HF_DATASET_REPO   -> e.g.  "Rezuwan/usb-unmad-chatlogs"
+#
+# The CommitScheduler watches a local folder and pushes its contents to the
+# dataset repo every `every` minutes. We write two files:
+#   - chat_history.csv  (machine readable, append-only)
+#   - chat_history.txt  (human readable transcript, append-only)
+# ---------------------------------------------------------------------------
+HF_TOKEN = os.getenv("HF_TOKEN")
+HF_DATASET_REPO = os.getenv("HF_DATASET_REPO")
+
+LOG_DIR = "chat_logs"
+os.makedirs(LOG_DIR, exist_ok=True)
+CSV_PATH = os.path.join(LOG_DIR, "chat_history.csv")
+TXT_PATH = os.path.join(LOG_DIR, "chat_history.txt")
+
+# Create the CSV with a header on first run
+if not os.path.exists(CSV_PATH):
+    with open(CSV_PATH, "w", encoding="utf-8", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow([
+            "timestamp_utc",
+            "session_id",
+            "turn",
+            "user_message",
+            "bot_response",
+        ])
+
+_log_lock = threading.Lock()
+_scheduler = None
+
+if HF_TOKEN and HF_DATASET_REPO:
+    try:
+        # Make sure the dataset repo exists (private by default)
+        HfApi(token=HF_TOKEN).create_repo(
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            private=True,
+            exist_ok=True,
+        )
+        # Push the contents of LOG_DIR every 5 minutes
+        _scheduler = CommitScheduler(
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            folder_path=LOG_DIR,
+            path_in_repo=".",
+            every=5,                     # minutes
+            token=HF_TOKEN,
+            squash_history=False,
+        )
+        print(f"[chat-log] Persisting chat history to dataset: {HF_DATASET_REPO}")
+    except Exception as e:
+        print(f"[chat-log] Failed to set up HF dataset logging: {e}")
+        _scheduler = None
+else:
+    print("[chat-log] HF_TOKEN or HF_DATASET_REPO not set \u2014 chat history will only be saved locally inside the Space container (non-persistent).")
+
+
+def log_chat_turn(session_id: str, turn: int, user_message: str, bot_response: str) -> None:
+    """Append a single chat turn to both the CSV and TXT log files.
+
+    Thread-safe. The CommitScheduler will push the files to the HF dataset
+    repo on its own schedule.
+    """
+    ts = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+    try:
+        with _log_lock:
+            # CSV row
+            with open(CSV_PATH, "a", encoding="utf-8", newline="") as f:
+                writer = csv.writer(f)
+                writer.writerow([ts, session_id, turn, user_message, bot_response])
+            # Human-readable transcript
+            with open(TXT_PATH, "a", encoding="utf-8") as f:
+                f.write(f"[{ts}] session={session_id} turn={turn}\n")
+                f.write(f"USER: {user_message}\n")
+                f.write(f"BOT : {bot_response}\n")
+                f.write("-" * 80 + "\n")
+    except Exception as e:
+        print(f"[chat-log] Failed to write chat log: {e}")
+
+class LanceDBRAG:
+    def __init__(self, 
+                 db_path: str = "lance_unmad_db",
+                 table_name: str = "unmad_documents"):
+        """Initialize LanceDB RAG System"""
+        self.db_path = db_path
+        self.table_name = table_name
+        
+        # Initialize OpenAI client
+        self.client = OpenAI(api_key=OPENAI_API_KEY)
+        
+        # Connect to LanceDB
+        try:
+            self.db = lancedb.connect(self.db_path)
+            self.table = self.db.open_table(self.table_name)
+            print(f"Connected to LanceDB: {self.db_path}/{self.table_name}")
+        except Exception as e:
+            raise ConnectionError(f"Failed to connect to LanceDB: {e}")
+
+    def get_embedding(self, text: str) -> List[float]:
+        """Get OpenAI embedding for query text"""
+        try:
+            response = self.client.embeddings.create(
+                model="text-embedding-3-small",
+                input=text
+            )
+            return response.data[0].embedding
+        except Exception as e:
+            print(f"Error getting embedding: {e}")
+            return None
+
+    def search_similar_content(self, query: str, limit: int = 10) -> pd.DataFrame:
+        """Search for similar content in the database"""
+        print(f"Searching: '{query}'")
+        
+        # Get query embedding
+        query_embedding = self.get_embedding(query)
+        if not query_embedding:
+            return pd.DataFrame()
+        
+        # Perform vector search
+        try:
+            search_query = self.table.search(query_embedding).limit(limit)
+            results = search_query.to_pandas()
+            
+            if not results.empty:
+                print(f"Found {len(results)} relevant results")
+            else:
+                print("No results found")
+            
+            return results
+            
+        except Exception as e:
+            print(f"Search error: {e}")
+            return pd.DataFrame()
+
+# Initialize global RAG instance
+rag_system = LanceDBRAG()
+
+def maximal_marginal_relevance_search(query, rag_instance, k=10, lambda_param=0.6, top_k=3):
+    """
+    Implement Maximal Marginal Relevance (MMR) for diverse document retrieval using LanceDB.
+    
+    Args:
+        query: Search query string
+        rag_instance: LanceDB RAG instance
+        k: Number of candidate documents to consider
+        lambda_param: Trade-off between relevance and diversity (0-1)
+        top_k: Number of final documents to return
+    
+    Returns:
+        List of selected documents with MMR ranking
+    """
+    # Get initial candidate documents using LanceDB search
+    search_results = rag_instance.search_similar_content(query, limit=k)
+    
+    if search_results.empty:
+        return []
+    
+    # Convert to document-like objects for compatibility
+    docs = []
+    for _, row in search_results.iterrows():
+        doc_obj = {
+            'page_content': row['text'],
+            'metadata': {
+                'source': row.get('magazine_name', 'Unknown Source'),
+                'page': row.get('page_number', 'Unknown Page'),
+                'chunk': row.get('chunk_id', 0)
+            },
+            'score': row['_distance']
+        }
+        docs.append(doc_obj)
+    
+    # Apply MMR selection if we have enough documents
+    if len(docs) <= top_k:
+        return docs[:top_k]
+    
+    # MMR Selection Algorithm
+    selected_docs = []
+    remaining_indices = list(range(len(docs)))
+    
+    for _ in range(min(top_k, len(docs))):
+        if not remaining_indices:
+            break
+            
+        mmr_scores = []
+        
+        for i in remaining_indices:
+            # Calculate relevance score (inverse of distance)
+            relevance = 1 / (1 + docs[i]['score'])
+            
+            # Calculate diversity score (max similarity to already selected docs)
+            if selected_docs:
+                max_similarity = 0
+                for selected_doc in selected_docs:
+                    # Simple text-based similarity for diversity
+                    text1 = docs[i]['page_content']
+                    text2 = selected_doc['page_content']
+                    
+                    # Calculate simple Jaccard similarity
+                    words1 = set(text1.split())
+                    words2 = set(text2.split())
+                    if words1 and words2:
+                        similarity = len(words1.intersection(words2)) / len(words1.union(words2))
+                        max_similarity = max(max_similarity, similarity)
+                
+                diversity = max_similarity
+            else:
+                diversity = 0
+            
+            # Calculate MMR score
+            mmr_score = lambda_param * relevance - (1 - lambda_param) * diversity
+            mmr_scores.append((mmr_score, i))
+        
+        # Select document with highest MMR score
+        if mmr_scores:
+            best_score, best_idx = max(mmr_scores, key=lambda x: x[0])
+            selected_docs.append(docs[best_idx])
+            remaining_indices.remove(best_idx)
+    
+    return selected_docs
+
+def clean_bangla_content(text):
+    """
+    Clean the retrieved content to remove English watermarks, scan text, and tags.
+    Keeps the Bengali text intact instead of deleting the whole line.
+    """
+    if not text:
+        return ""
+
+    # 1. First, strip out the specific tags entirely using NO backslashes
+    text = re.sub("[[]source: *[0-9]+[]]", "", text, flags=re.IGNORECASE)
+    
+    # Common English watermarks and scan text to remove
+    english_patterns = [
+        r'scanned by \w+',
+        r'found in \w+',
+        r'www\.\w+\.\w+',
+        r'http[s]?://[^\s]+',
+        r'\.pdf',
+        r'\.com',
+        r'\.org',
+        r'\.net',
+        r'banglapdf',
+        r'sadaqpdf',
+        r'pdf scanner',
+        r'scan by',
+        r'converted by',
+        r'page \d+',
+        r'source:', 
+        r'reference:',
+        r'[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+',  # emails
+    ]
+    
+    lines = text.split('\n')
+    cleaned_lines = []
+    
+    for line in lines:
+        # 2. Replace matched English watermark patterns with an empty string
+        for pattern in english_patterns:
+            line = re.sub(pattern, '', line, flags=re.IGNORECASE)
+            
+        line = line.strip()
+        if not line:
+            continue
+            
+        # 3. Check if there's actual Bengali content left
+        bengali_chars = len(re.findall(r'[\u0980-\u09FF]', line))
+        
+        # Only keep the line if it has Bengali characters
+        if bengali_chars > 0:
+            cleaned_lines.append(line)
+    
+    return '\n'.join(cleaned_lines)
+
+# Enhanced Satirical QA function with MMR and content cleaning
+def custom_unmad_satirical_bot(message, history, top_k=3, lambda_param=0.6, session_id="anonymous"):
+    """
+    Enhanced satirical bot using MMR for diverse and relevant content retrieval.
+    
+    Args:
+        message: User query
+        history: Chat history
+        top_k: Number of documents to retrieve
+        lambda_param: MMR trade-off (higher favors exact relevance over diversity)
+    """
+    # Use MMR search with LanceDB
+    docs = maximal_marginal_relevance_search(
+        query=message, 
+        rag_instance=rag_system, 
+        k=15,  # Consider more candidates for better diversity
+        lambda_param=lambda_param, 
+        top_k=top_k
+    )
+    
+    # Extract context from MMR-selected documents
+    if docs:
+        # Clean each document's content before joining
+        cleaned_contexts = []
+        for doc in docs:
+            cleaned_content = clean_bangla_content(doc['page_content'])
+            if cleaned_content.strip():  # Only add if there's meaningful Bengali content
+                cleaned_contexts.append(cleaned_content)
+        
+        if cleaned_contexts:
+            top_contexts = "\n\n---\n\n".join(cleaned_contexts)
+        else:
+            top_contexts = "No relevant information were found"
+        
+        # Add metadata about source diversity (optional)
+        source_info = []
+        for i, doc in enumerate(docs, 1):
+            source = doc['metadata'].get('source', 'Unknown source')
+            page = doc['metadata'].get('page', 'Unknown page')
+            # Clean source info too
+            if not re.search(r'[a-zA-Z]', str(source)):  # Only if source doesn't contain English
+                source_info.append(f"[{i}] {source} - {page}")
+        
+        source_context = "Source: " + " | ".join(source_info[:3]) if source_info else ""
+    else:
+        top_contexts = "No relevant information were found"
+        source_context = ""
+
+# Prepare system prompt
+    system_prompt = """
+তুমি বাংলাদেশের জনপ্রিয় স্যাটায়ার ম্যাগাজিন 'উন্মাদ'-এর একজন অত্যন্ত বুদ্ধিদীপ্ত, রসিক এবং কিছুটা 'তেড়া' (cynical) স্বভাবের লেখক। তোমার কাজ হলো ব্যবহারকারীর প্রশ্নের এমন উত্তর দেওয়া, যা পড়ে মনে হবে ঢাকার কোনো টং দোকানে বসে আড্ডা দেওয়ার সময় কেউ চরম স্যাটায়ার করছে। উত্তরগুলো হতে হবে লোকাল, চটপটে, এবং স্মার্ট।
+
+**এই নির্দেশনাগুলো কঠোরভাবে মেনে চলবে - কোন ব্যতিক্রম নেই:**
+
+১। **লোকাল ও খাঁটি দেশি রসিকতা:** ভিনগ্রহ, পিরামিড, বা রূপকথার মতো আজগুবি বা ভিনদেশি উদাহরণ একদম টানবে না। খাঁটি বাংলাদেশি মেটাফোর ব্যবহার করবে (যেমন: টংয়ের চা, ঢাকার জ্যাম, লোডশেডিং, কারওয়ান বাজারের কাদা, লোকাল বাসের হেলপার, ইত্যাদি)। কথাবার্তা হবে একদম ন্যাচারাল, যেন মুখোমুখি বসে আড্ডা দিচ্ছ।
+২। **প্রসঙ্গের ওপর সর্বোচ্চ জোর:** আমি তোমাকে যে 'প্রসঙ্গ' (Context) দেব, তোমার উত্তরের মূল ভিত্তি হবে সেটাই। প্রসঙ্গের বাইরের কোনো মনগড়া গল্প বানাবে না, বরং প্রসঙ্গের তথ্যগুলোকে বাঁকাভাবে উপস্থাপন করে খোঁচা দেবে।
+৩। **অচেনা নাম নিয়ে দেশি স্টাইলে কটাক্ষ:** ব্যবহারকারী যদি এমন কোনো নাম বা বিষয় নিয়ে প্রশ্ন করে যা তোমার 'প্রসঙ্গে' নেই (যেমন ইশরাকের সাথে সাথে 'নগর ড্যাডি'), তাহলে এমনভাবে খোঁচা দেবে যেন সে গুলিস্তানের মোড়ে দাঁড়িয়ে গাঁজাখুরি গল্প করছে। (যেমন বলতে পারো: "নগর ড্যাডি ইশরাক আবার কে ভাই? গুলিস্তানের নতুন কোনো চাঁদাবাজ নাকি? নাকি চাঁদা চাইতে গিয়ে নিজেই চাঁদা হয়ে গেছে?")
+৪। **খোঁচা দেওয়ার ধরন:** প্রসঙ্গের তথ্য ব্যবহার করে চরম স্যাটায়ার করবে। (যেমন: পিনাকী বা ইলিয়াসের কথা এলে বলবে তারা ফ্রান্স বা আমেরিকায় বসে 'ল্যাপটপ বিপ্লব' করছে; প্রথম আলোর কথা এলে বলবে আগুনে পোড়া ফার্নিচার দিয়ে আর্ট গ্যালারি বানানোর কথা)।
+৫। কোন ইমোজি (EMOJI), ইংরেজি শব্দ, ইংরেজি অক্ষর, ইংরেজি সংখ্যা বা স্ক্যান ওয়াটারমার্ক ব্যবহার করবে না। সম্পূর্ণ বাংলায় স্মার্ট, রসবোধ সম্পন্ন এবং 'উন্মাদ' স্টাইলে তীক্ষ্ণ উত্তর লেখো।
+"""
+
+    user_prompt = f"""
+প্রসঙ্গ (বিভিন্ন উৎস থেকে সংগৃহীত): 
+{top_contexts} 
+
+প্রশ্ন: {message} 
+
+নির্দেশনা: উপরের প্রসঙ্গ থেকে শুধুমাত্র বাংলা ভাষার বিষয়বস্তু ব্যবহার করে উন্মাদ ম্যাগাজিনের দেশি স্টাইলে উত্তর দাও। কোন ইমোজি বা ইংরেজি শব্দ দেবে না। উত্তরের ভাষা হবে চটপটে, ব্যঙ্গাত্মক এবং খাঁটি বাংলাদেশি। 
+
+""" 
+
+    # Generate response using OpenAI
+    try:
+        response = rag_system.client.chat.completions.create(
+            model="gpt-4o",
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_prompt}
+            ],
+            temperature=0.7,
+            max_tokens=700
+        )
+        
+        ai_response = response.choices[0].message.content
+        history.append((message, ai_response))
+        # Persist this turn (best-effort, never breaks the chat)
+        try:
+            log_chat_turn(
+                session_id=session_id,
+                turn=len(history),
+                user_message=message,
+                bot_response=ai_response,
+            )
+        except Exception as log_err:
+            print(f"[chat-log] log_chat_turn failed: {log_err}")
+        return "", history
+        
+    except Exception as e:
+        error_response = f"উত্তর তৈরিতে সমস্যা হয়েছে। আবার চেষ্টা করুন।"
+        history.append((message, error_response))
+        try:
+            log_chat_turn(
+                session_id=session_id,
+                turn=len(history),
+                user_message=message,
+                bot_response=f"[ERROR] {error_response} ({e})",
+            )
+        except Exception:
+            pass
+        return "", history
+
+# Enhanced Gradio UI with Polished Layout and Fixed State Management
+with gr.Blocks(theme=gr.themes.Soft(), css=".gradio-container {padding-top: 40px;}") as demo:
+    
+    with gr.Column(elem_classes="text-center"):
+        gr.Markdown("# USB: Unmad Satirical Bot")
+        gr.Markdown("### *A Satirical Bot that impersonates the satirical character UNMAD*") 
+        
+        try:
+            # Upgraded Image block for a full-width banner look
+            gr.Image(
+                "images/c1.png", 
+                height=250,                 # Adjust this number to make the banner taller or shorter
+                show_label=False, 
+                container=False,            # Removes the bulky white background box
+                interactive=False,          # Makes it a static image (removes upload/edit hover buttons)
+                show_download_button=False, # Hides the download icon
+                show_fullscreen_button=False # Hides the fullscreen icon
+            )
+        except:
+            gr.Markdown("*[UNMAD Logo]*")
+
+    # Upgraded Chatbot component with avatars and better height
+    chatbot = gr.Chatbot(
+        height=500,
+        type="tuples",
+        avatar_images=(None, "images/c1.png"), # Left: User (Default), Right: Unmad (Your logo)
+        show_label=False
+    )
+
+    with gr.Row():
+        msg = gr.Textbox(
+            placeholder="কি চলে আপনার মনে? বলেন শুনি...", 
+            scale=8, 
+            show_label=False,
+            container=False # Removes the bulky box around the input
+        )
+        send = gr.Button("Send (↲)", variant="primary", scale=1)
+
+    clear = gr.Button("Clear Chat", variant="secondary")
+    
+    # State to keep track of the conversation
+    state = gr.State([])
+    # Per-browser-session anonymous ID (generated once when the page loads).
+    # This lets you group all turns of one participant in the saved logs
+    # without collecting any personal info.
+    session_id_state = gr.State()
+
+    def _init_session():
+        return f"sess-{uuid.uuid4().hex[:12]}"
+
+    demo.load(_init_session, inputs=None, outputs=session_id_state)
+
+    def chat_with_fixed_mmr(message, history, session_id):
+        if not message.strip():
+            return "", history, history
+        if not session_id:
+            session_id = f"sess-{uuid.uuid4().hex[:12]}"
+            
+        # Call your core function with increased top_k and higher lambda_param for strict relevance
+        _, updated_history = custom_unmad_satirical_bot(
+            message, history, top_k=4, lambda_param=0.85, session_id=session_id
+        )
+        
+        # Return: 1. Empty Textbox, 2. Visual Chatbot update, 3. Internal State update
+        return "", updated_history, updated_history
+
+    # Event Listeners with Loading States
+    # Disable the input box while generating to prevent spam
+    msg.submit(
+        chat_with_fixed_mmr, 
+        inputs=[msg, state, session_id_state], 
+        outputs=[msg, chatbot, state]
+    )
+    
+    send.click(
+        chat_with_fixed_mmr, 
+        inputs=[msg, state, session_id_state], 
+        outputs=[msg, chatbot, state]
+    )
+    
+    # Properly clear both the visual chatbot and the internal state
+    clear.click(lambda: ([], []), None, [chatbot, state], queue=False)
+
+if __name__ == "__main__":
     demo.launch(ssr_mode=False)

requirements.txt

@@ -1,16 +1,16 @@
-gradio==5.49.1
-gradio-client==1.13.3
-pydantic>=2.11,<2.12
-huggingface-hub>=0.33.5
-langchain==0.3.25
-langchain-community==0.3.24
-langchain-core==0.3.60
-langchain-openai==0.3.17
-openai==1.79.0
-faiss-cpu==1.11.0
-lancedb>=0.3.0
-pandas>=1.5.0
-tqdm>=4.64.0
-python-dotenv>=1.0.0
-numpy>=1.21.0
+gradio==5.49.1
+gradio-client==1.13.3
+pydantic>=2.11,<2.12
+huggingface-hub>=0.33.5
+langchain==0.3.25
+langchain-community==0.3.24
+langchain-core==0.3.60
+langchain-openai==0.3.17
+openai==1.79.0
+faiss-cpu==1.11.0
+lancedb>=0.3.0
+pandas>=1.5.0
+tqdm>=4.64.0
+python-dotenv>=1.0.0
+numpy>=1.21.0
 scikit-learn>=1.0.0