app.py

import gradio as gr
import os
import sqlite3
import json
import hashlib
from datetime import datetime
from typing import List, Dict, Any, Tuple, Optional
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import threading

from utils import (
    process_document,
    extract_axioms,
    generate_response,
    get_embedding,
    compute_similarity,
    Document,
    Axiom,
    ActivityLog
)

# Initialize database
DB_PATH = "rag_nexus.db"
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
cursor = conn.cursor()

# Create tables
cursor.execute("""
CREATE TABLE IF NOT EXISTS documents (
    id TEXT PRIMARY KEY,
    name TEXT,
    content TEXT,
    size INTEGER,
    uploaded_at TEXT,
    chunk_count INTEGER
)
""")

cursor.execute("""
CREATE TABLE IF NOT EXISTS axioms (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    doc_id TEXT,
    source TEXT,
    axiom TEXT,
    confidence REAL,
    FOREIGN KEY (doc_id) REFERENCES documents (id)
)
""")

cursor.execute("""
CREATE TABLE IF NOT EXISTS activity (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    action TEXT,
    details TEXT,
    timestamp TEXT
)
""")

conn.commit()

# Thread-local storage for database connections
thread_local = threading.local()

def get_db():
    """Get thread-local database connection"""
    if not hasattr(thread_local, 'conn'):
        thread_local.conn = sqlite3.connect(DB_PATH)
    return thread_local.conn

class RAGState:
    def __init__(self):
        self.vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
        self.document_chunks = []
        self.chunk_metadata = []
        self.is_initialized = False
        
    def initialize_models(self):
        """Initialize models (simulated)"""
        if not self.is_initialized:
            # Load existing documents
            conn = get_db()
            cursor = conn.cursor()
            cursor.execute("SELECT id, content FROM documents")
            docs = cursor.fetchall()
            
            if docs:
                chunks = []
                metadata = []
                for doc_id, content in docs:
                    doc_chunks = [content[i:i+500] for i in range(0, len(content), 500)]
                    chunks.extend(doc_chunks)
                    metadata.extend([{"doc_id": doc_id, "chunk_idx": i} for i in range(len(doc_chunks))])
                
                if chunks:
                    self.vectorizer.fit(chunks)
                    self.document_chunks = chunks
                    self.chunk_metadata = metadata
            
            self.is_initialized = True

def get_state():
    """Get global state"""
    if not hasattr(get_state, 'state'):
        get_state.state = RAGState()
    return get_state.state

def log_activity(action: str, details: Dict[str, Any]):
    """Log activity to database"""
    conn = get_db()
    cursor = conn.cursor()
    cursor.execute(
        "INSERT INTO activity (action, details, timestamp) VALUES (?, ?, ?)",
        (action, json.dumps(details), datetime.now().isoformat())
    )
    conn.commit()

def get_stats():
    """Get system statistics"""
    conn = get_db()
    cursor = conn.cursor()
    
    cursor.execute("SELECT COUNT(*) FROM documents")
    doc_count = cursor.fetchone()[0]
    
    cursor.execute("SELECT COUNT(*) FROM axioms")
    axiom_count = cursor.fetchone()[0]
    
    cursor.execute("SELECT SUM(size) FROM documents")
    storage = cursor.fetchone()[0] or 0
    
    return {
        "doc_count": doc_count,
        "axiom_count": axiom_count,
        "storage_mb": round(storage / 1024 / 1024, 2)
    }

def load_documents():
    """Load all documents"""
    conn = get_db()
    cursor = conn.cursor()
    cursor.execute("SELECT id, name, size, uploaded_at FROM documents ORDER BY uploaded_at DESC")
    docs = cursor.fetchall()
    
    if not docs:
        return [["No documents found", "", "", ""]]
    
    return [[doc[1], f"{doc[2]} bytes", doc[3], doc[0]] for doc in docs]

def load_axioms(source_filter: str = ""):
    """Load axioms with optional source filter"""
    conn = get_db()
    cursor = conn.cursor()
    
    if source_filter:
        cursor.execute("""
            SELECT a.id, a.source, a.axiom, a.confidence, d.name 
            FROM axioms a 
            JOIN documents d ON a.doc_id = d.id 
            WHERE d.name LIKE ? 
            ORDER BY a.confidence DESC
        """, (f"%{source_filter}%",))
    else:
        cursor.execute("""
            SELECT a.id, a.source, a.axiom, a.confidence, d.name 
            FROM axioms a 
            JOIN documents d ON a.doc_id = d.id 
            ORDER BY a.confidence DESC
        """)
    
    axioms = cursor.fetchall()
    
    if not axioms:
        return [["No axioms found", "", "", "", ""]]
    
    return [[ax[4], ax[1], ax[2][:100] + "...", f"{ax[3]:.2f}", str(ax[0])] for ax in axioms]

def load_activity():
    """Load recent activity"""
    conn = get_db()
    cursor = conn.cursor()
    cursor.execute("SELECT action, details, timestamp FROM activity ORDER BY timestamp DESC LIMIT 20")
    activities = cursor.fetchall()
    
    if not activities:
        return [["No activity yet", "", ""]]
    
    return [[act[0], json.loads(act[1]).get('description', ''), act[2]] for act in activities]

def process_uploaded_files(files: List[str]) -> Tuple[str, str]:
    """Process uploaded files and return status"""
    if not files:
        return "No files uploaded", "⚠️"
    
    state = get_state()
    success_count = 0
    total_count = len(files)
    
    for file_path in files:
        try:
            # Process document
            doc = process_document(file_path)
            
            # Save to database
            conn = get_db()
            cursor = conn.cursor()
            cursor.execute(
                "INSERT INTO documents (id, name, content, size, uploaded_at, chunk_count) VALUES (?, ?, ?, ?, ?, ?)",
                (doc.id, doc.name, doc.content, doc.size, doc.uploaded_at, doc.chunk_count)
            )
            
            # Extract axioms
            axioms = extract_axioms(doc.content, doc.id)
            for axiom in axioms:
                cursor.execute(
                    "INSERT INTO axioms (doc_id, source, axiom, confidence) VALUES (?, ?, ?, ?)",
                    (doc.id, axiom.source, axiom.text, axiom.confidence)
                )
            
            conn.commit()
            
            # Update vector store
            chunks = [doc.content[i:i+500] for i in range(0, len(doc.content), 500)]
            state.document_chunks.extend(chunks)
            state.chunk_metadata.extend([{"doc_id": doc.id, "chunk_idx": i} for i in range(len(chunks))])
            
            # Refit vectorizer if needed
            if state.document_chunks:
                state.vectorizer.fit(state.document_chunks)
            
            log_activity("document_uploaded", {
                "name": doc.name,
                "size": doc.size,
                "chunks": doc.chunk_count
            })
            
            success_count += 1
            
        except Exception as e:
            log_activity("upload_failed", {
                "file": os.path.basename(file_path),
                "error": str(e)
            })
    
    # Clean up temporary files
    for file_path in files:
        try:
            os.unlink(file_path)
        except:
            pass
    
    return f"Processed {success_count}/{total_count} files", "✅" if success_count == total_count else "⚠️"

def generate_rag_response(query: str, use_axioms: bool, use_context: bool) -> Tuple[str, str]:
    """Generate response using RAG"""
    if not query.strip():
        return "Please enter a query", ""
    
    state = get_state()
    state.initialize_models()
    
    # Retrieve context
    context = ""
    retrieved_docs = []
    
    if use_context and state.document_chunks:
        try:
            query_vec = state.vectorizer.transform([query])
            doc_vecs = state.vectorizer.transform(state.document_chunks)
            similarities = cosine_similarity(query_vec, doc_vecs).flatten()
            
            # Get top 3 chunks
            top_indices = np.argsort(similarities)[-3:][::-1]
            
            for idx in top_indices:
                if similarities[idx] > 0.1:
                    chunk = state.document_chunks[idx]
                    doc_id = state.chunk_metadata[idx]["doc_id"]
                    conn = get_db()
                    cursor = conn.cursor()
                    cursor.execute("SELECT name FROM documents WHERE id = ?", (doc_id,))
                    doc_name = cursor.fetchone()[0]
                    
                    context += f"\n\n--- From {doc_name} ---\n{chunk}"
                    retrieved_docs.append(f"{doc_name} (similarity: {similarities[idx]:.2f})")
        except:
            context = ""
            retrieved_docs = ["No relevant context found"]
    
    # Get axioms
    axioms = []
    if use_axioms:
        conn = get_db()
        cursor = conn.cursor()
        cursor.execute("SELECT axiom FROM axioms ORDER BY RANDOM() LIMIT 5")
        axioms = [row[0] for row in cursor.fetchall()]
    
    # Generate response
    response = generate_response(query, context, axioms)
    
    # Log activity
    log_activity("response_generated", {
        "query": query[:100],
        "used_axioms": use_axioms,
        "used_context": use_context
    })
    
    # Format context info
    context_info = "\n".join(retrieved_docs) if retrieved_docs else "No context retrieved"
    
    return response, context_info

def clear_all_data():
    """Clear all data from database"""
    conn = get_db()
    cursor = conn.cursor()
    cursor.execute("DELETE FROM documents")
    cursor.execute("DELETE FROM axioms")
    cursor.execute("DELETE FROM activity")
    conn.commit()
    
    # Reset state
    state = get_state()
    state.document_chunks = []
    state.chunk_metadata = []
    
    log_activity("data_cleared", {"all": True})
    
    return "All data cleared successfully", "✅"

def export_axioms():
    """Export axioms as JSON"""
    conn = get_db()
    cursor = conn.cursor()
    cursor.execute("""
        SELECT d.name as document, a.source, a.axiom, a.confidence 
        FROM axioms a 
        JOIN documents d ON a.doc_id = d.id
    """)
    axioms = [{"document": row[0], "source": row[1], "axiom": row[2], "confidence": row[3]} 
              for row in cursor.fetchall()]
    
    if not axioms:
        return "No axioms to export", "⚠️"
    
    filename = f"axioms_export_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
    with open(filename, 'w') as f:
        json.dump(axioms, f, indent=2)
    
    log_activity("axioms_exported", {"count": len(axioms), "file": filename})
    
    return f"Exported {len(axioms)} axioms to {filename}", "✅"

# Initialize app state on load
def initialize_app():
    state = get_state()
    state.initialize_models()
    return "✅ Models initialized"

# Create Gradio interface
with gr.Blocks() as demo:
    gr.Markdown(
        """
        # 🔮 RAG Nexus
        ### Intelligent Document Analysis & Axiom Extraction System
        **Built with anycoder** | [View on Hugging Face](https://huggingface.co/spaces/akhaliq/anycoder)
        """
    )
    
    # Status bar
    with gr.Row():
        status_text = gr.Textbox("Initializing...", label="System Status", scale=4)
        init_btn = gr.Button("🔄 Reinitialize", scale=1)
    
    # Tabs
    with gr.Tabs() as tabs:
        # Upload Tab
        with gr.TabItem("📤 Upload", id="upload"):
            gr.Markdown("### Upload Documents for Analysis")
            
            file_output = gr.File(
                label="Drop files here or click to browse",
                file_count="multiple",
                file_types=[".txt", ".md", ".pdf", ".doc", ".docx"]
            )
            
            upload_btn = gr.Button("🚀 Process Files", variant="primary")
            upload_status = gr.Textbox(label="Upload Status", interactive=False)
            
            with gr.Accordion("📋 Upload Queue", open=False):
                upload_queue = gr.Dataframe(
                    headers=["File", "Status", "Size (bytes)"],
                    datatype=["str", "str", "number"],
                    label="Processed Files"
                )
        
        # Documents Tab
        with gr.TabItem("📚 Documents", id="documents"):
            gr.Markdown("### Indexed Documents")
            
            with gr.Row():
                doc_search = gr.Textbox(
                    placeholder="Search documents...",
                    label="Search",
                    scale=3
                )
                clear_docs_btn = gr.Button("🗑️ Clear All", variant="stop", scale=1)
            
            documents_table = gr.Dataframe(
                headers=["Name", "Size", "Uploaded", "ID"],
                datatype=["str", "str", "str", "str"],
                label="Documents",
                wrap=True
            )
            
            doc_search.change(
                fn=lambda search: load_documents(),
                inputs=doc_search,
                outputs=documents_table,
                api_visibility="private"
            )
        
        # Axioms Tab
        with gr.TabItem("⚡ Axioms", id="axioms"):
            gr.Markdown("### Extracted Axioms")
            
            with gr.Row():
                axiom_search = gr.Textbox(
                    placeholder="Search axioms...",
                    label="Search",
                    scale=2
                )
                axiom_filter = gr.Dropdown(
                    choices=[],
                    label="Filter by Document",
                    scale=1
                )
                export_axioms_btn = gr.Button("💾 Export JSON", scale=1)
            
            axioms_table = gr.Dataframe(
                headers=["Document", "Source", "Axiom", "Confidence", "ID"],
                datatype=["str", "str", "str", "number", "str"],
                label="Axioms",
                wrap=True
            )
            
            export_status = gr.Textbox(label="Export Status", interactive=False)
        
        # Generate Tab
        with gr.TabItem("🤖 Generate", id="generate"):
            gr.Markdown("### Intelligent Response Generation")
            
            query_input = gr.Textbox(
                label="Enter your query",
                placeholder="Ask anything about your documents... (e.g., 'What are the fundamental principles based on the uploaded documents?')",
                lines=4,
                max_lines=8
            )
            
            with gr.Row():
                use_axioms = gr.Checkbox(label="Use Axioms", value=True)
                use_context = gr.Checkbox(label="Use Context (RAG)", value=True)
            
            generate_btn = gr.Button("🚀 Generate Response", variant="primary")
            
            with gr.Group():
                response_output = gr.Markdown(
                    label="Generated Response",
                    show_copy_button=True
                )
                
                with gr.Accordion("📚 Retrieved Context & Axioms", open=False):
                    context_output = gr.Textbox(
                        label="Retrieved Documents",
                        lines=5,
                        interactive=False
                    )
            
            query_stats = gr.Textbox(
                label="Query Statistics",
                interactive=False,
                visible=False
            )
        
        # Analytics Tab
        with gr.TabItem("📊 Analytics", id="analytics"):
            gr.Markdown("### System Analytics")
            
            with gr.Row():
                with gr.Column():
                    doc_count_label = gr.Label(value="0", label="📄 Documents", show_label=True)
                with gr.Column():
                    axiom_count_label = gr.Label(value="0", label="⚡ Axioms", show_label=True)
                with gr.Column():
                    storage_label = gr.Label(value="0MB", label="💾 Storage Used", show_label=True)
            
            with gr.Accordion("📈 Recent Activity", open=True):
                activity_log = gr.Dataframe(
                    headers=["Action", "Details", "Timestamp"],
                    datatype=["str", "str", "str"],
                    label="Activity Log",
                    wrap=True,
                    max_height=300
                )
    
    # Event handlers
    init_btn.click(
        fn=initialize_app,
        outputs=status_text,
        api_visibility="private"
    )
    
    # Upload events
    def process_and_update(files):
        if not files:
            return "No files selected", []
        
        # Process files
        status, icon = process_uploaded_files(files)
        
        # Create queue table
        queue_data = []
        for f in files:
            name = os.path.basename(f)
            size = os.path.getsize(f) if os.path.exists(f) else 0
            queue_data.append([name, "✅ Processed", size])
        
        return f"{icon} {status}", queue_data
    
    upload_btn.click(
        fn=process_and_update,
        inputs=file_output,
        outputs=[upload_status, upload_queue],
        api_visibility="private"
    ).then(
        fn=load_documents,
        outputs=documents_table
    ).then(
        fn=lambda: load_axioms(),
        outputs=axioms_table
    ).then(
        fn=get_stats,
        outputs=[doc_count_label, axiom_count_label, storage_label]
    ).then(
        fn=load_activity,
        outputs=activity_log
    )
    
    # Documents tab events
    def refresh_documents():
        docs = load_documents()
        # Update filter choices
        return docs
    
    tabs.change(
        fn=refresh_documents,
        outputs=documents_table,
        api_visibility="private"
    )
    
    clear_docs_btn.click(
        fn=clear_all_data,
        outputs=[status_text],
        api_visibility="private"
    ).then(
        fn=load_documents,
        outputs=documents_table
    ).then(
        fn=lambda: load_axioms(),
        outputs=axioms_table
    ).then(
        fn=get_stats,
        outputs=[doc_count_label, axiom_count_label, storage_label]
    )
    
    # Axioms tab events
    def update_axiom_filter():
        conn = get_db()
        cursor = conn.cursor()
        cursor.execute("SELECT DISTINCT name FROM documents")
        docs = [row[0] for row in cursor.fetchall()]
        return gr.Dropdown(choices=[""] + docs)
    
    tabs.change(
        fn=update_axiom_filter,
        outputs=axiom_filter,
        api_visibility="private"
    )
    
    axiom_filter.change(
        fn=lambda filter_val: load_axioms(filter_val or ""),
        inputs=axiom_filter,
        outputs=axioms_table,
        api_visibility="private"
    )
    
    export_axioms_btn.click(
        fn=export_axioms,
        outputs=[export_status],
        api_visibility="private"
    )
    
    # Generate tab events
    generate_btn.click(
        fn=generate_rag_response,
        inputs=[query_input, use_axioms, use_context],
        outputs=[response_output, context_output],
        api_visibility="private"
    ).then(
        fn=load_activity,
        outputs=activity_log
    )
    
    # Load initial data
    demo.load(
        fn=initialize_app,
        outputs=status_text,
        api_visibility="private"
    ).then(
        fn=load_documents,
        outputs=documents_table
    ).then(
        fn=lambda: load_axioms(),
        outputs=axioms_table
    ).then(
        fn=get_stats,
        outputs=[doc_count_label, axiom_count_label, storage_label]
    ).then(
        fn=load_activity,
        outputs=activity_log
    ).then(
        fn=update_axiom_filter,
        outputs=axiom_filter
    )

# Launch with Gradio 6 theme
demo.launch(
    theme=gr.themes.Soft(
        primary_hue="indigo",
        secondary_hue="violet",
        neutral_hue="slate",
        font=gr.themes.GoogleFont("Inter"),
        text_size="lg",
        spacing_size="lg",
        radius_size="md"
    ).set(
        button_primary_background_fill="*primary_600",
        button_primary_background_fill_hover="*primary_700",
        block_title_text_weight="600",
        block_background_fill="*neutral_50"
    ),
    footer_links=[{"label": "Built with anycoder", "url": "https://huggingface.co/spaces/akhaliq/anycoder"}],
    show_error=True,
    max_threads=40
)