Update app.py

app.py

@@ -1,357 +1,353 @@
-import gradio as gr
-import os
-import hashlib
-import json
-import pickle
-from datetime import datetime, timedelta
-from pathlib import Path
-from dotenv import load_dotenv
-
-# Import your original RAG technique modules
-from Hyde import get_answer_using_hyde
-from QueryDecomposition import get_answer_using_query_decomposition
-from QueryExpansion import get_answer_using_query_expansion
-from RagFusion import get_answer_using_rag_fusion
-from StepBackQuery import get_answer
-
-# Import new advanced retrieval techniques
-from AdvancedRag import (
-    get_answer_using_multi_query,
-    get_answer_using_parent_child,
-    get_answer_using_contextual_compression,
-    get_answer_using_cross_encoder,
-    get_answer_using_semantic_routing
-)
-
-load_dotenv()
-
-# Cache configuration
-CACHE_DIR = Path("rag_cache")
-CACHE_DIR.mkdir(exist_ok=True)
-CACHE_EXPIRY_HOURS = 24  # Cache expires after 24 hours
-
-# Extended dictionary mapping technique names to their corresponding functions
-RAG_TECHNIQUES = {
-    # Original Techniques
-    "HyDE (Hypothetical Document Embeddings)": get_answer_using_hyde,
-    "Query Decomposition": get_answer_using_query_decomposition,
-    "Query Expansion": get_answer_using_query_expansion,
-    "RAG Fusion": get_answer_using_rag_fusion,
-    "Step Back Query": get_answer,
-    
-    # Advanced Retrieval Techniques
-    "Multi-Query Retrieval": get_answer_using_multi_query,
-    "Parent-Child Retrieval": get_answer_using_parent_child,
-    "Contextual Compression": get_answer_using_contextual_compression,
-    "Cross-Encoder Reranking": get_answer_using_cross_encoder,
-    "Semantic Routing": get_answer_using_semantic_routing,
-}
-
-def generate_cache_key(link, technique):
-    """
-    Generate a unique cache key based on link and technique
-    """
-    cache_string = f"{link}_{technique}"
-    return hashlib.md5(cache_string.encode()).hexdigest()
-
-def get_cache_file_path(cache_key):
-    """
-    Get the full path for a cache file
-    """
-    return CACHE_DIR / f"{cache_key}.pkl"
-
-def is_cache_valid(cache_file_path):
-    """
-    Check if cache file exists and is not expired
-    """
-    if not cache_file_path.exists():
-        return False
-    
-    # Check if cache is expired
-    file_time = datetime.fromtimestamp(cache_file_path.stat().st_mtime)
-    expiry_time = datetime.now() - timedelta(hours=CACHE_EXPIRY_HOURS)
-    
-    return file_time > expiry_time
-
-def save_to_cache(cache_key, data):
-    """
-    Save data to cache file
-    """
-    try:
-        cache_file_path = get_cache_file_path(cache_key)
-        cache_data = {
-            'data': data,
-            'timestamp': datetime.now().isoformat(),
-            'cache_key': cache_key
-        }
-        
-        with open(cache_file_path, 'wb') as f:
-            pickle.dump(cache_data, f)
-        
-        print(f"✅ Cached result for key: {cache_key}")
-        return True
-    except Exception as e:
-        print(f"❌ Failed to save cache: {e}")
-        return False
-
-def load_from_cache(cache_key):
-    """
-    Load data from cache file
-    """
-    try:
-        cache_file_path = get_cache_file_path(cache_key)
-        
-        if not is_cache_valid(cache_file_path):
-            return None
-        
-        with open(cache_file_path, 'rb') as f:
-            cache_data = pickle.load(f)
-        
-        print(f"🎯 Cache hit for key: {cache_key}")
-        return cache_data['data']
-    except Exception as e:
-        print(f"❌ Failed to load cache: {e}")
-        return None
-
-def clear_expired_cache():
-    """
-    Automatically clear expired cache files
-    """
-    try:
-        cache_files = list(CACHE_DIR.glob("*.pkl"))
-        expired_count = 0
-        
-        for cache_file in cache_files:
-            if not is_cache_valid(cache_file):
-                cache_file.unlink()
-                expired_count += 1
-        
-        if expired_count > 0:
-            print(f"🧹 Auto-cleared {expired_count} expired cache files")
-    except Exception as e:
-        print(f"❌ Failed to auto-clear expired cache: {e}")
-
-def process_rag_query(link, question, technique):
-    """
-    Process the RAG query using the selected technique with caching
-    """
-    try:
-        if not link or not question:
-            return "Please provide both a link and a question."
-        
-        if not link.startswith(('http://', 'https://')):
-            return "Please provide a valid URL starting with http:// or https://"
-        
-        # Auto-clear expired cache files
-        clear_expired_cache()
-        
-        # Generate cache key based on link and technique
-        cache_key = generate_cache_key(link, technique)
-        
-        # Try to load from cache first
-        cached_result = load_from_cache(cache_key)
-        if cached_result is not None:
-            # Check if we have this specific question cached
-            if isinstance(cached_result, dict) and question in cached_result:
-                return cached_result[question]
-        
-        # Get the corresponding function for the selected technique
-        rag_function = RAG_TECHNIQUES.get(technique)
-        
-        if not rag_function:
-            return "Invalid technique selected."
-        
-        print(f"🔄 Processing new query: {technique} for {link}")
-        
-        # Call the appropriate RAG function
-        answer = rag_function(link, question)
-        
-        # Save to cache
-        if cached_result is None:
-            cached_result = {}
-        elif not isinstance(cached_result, dict):
-            cached_result = {}
-        
-        cached_result[question] = answer
-        save_to_cache(cache_key, cached_result)
-        
-        return answer
-        
-    except Exception as e:
-        return f"Error processing query: {str(e)}\n\nNote: Advanced techniques require additional dependencies. Make sure you have installed: sentence-transformers, scikit-learn"
-
-def create_webpage_preview(link):
-    """
-    Create an HTML iframe to preview the webpage
-    """
-    if not link:
-        return ""
-    
-    if not link.startswith(('http://', 'https://')):
-        return "<p style='color: red;'>Please provide a valid URL starting with http:// or https://</p>"
-    
-    # Create an iframe to display the webpage
-    iframe_html = f"""
-    <div style="width: 100%; height: 500px; border: 1px solid #ccc; border-radius: 5px;">
-        <iframe src="{link}" width="100%" height="100%" frameborder="0" 
-                style="border-radius: 5px;">
-            <p>Your browser does not support iframes. 
-            <a href="{link}" target="_blank">Click here to open the link</a></p>
-        </iframe>
-    </div>
-    """
-    return iframe_html
-
-# Create the Gradio interface
-def create_interface():
-    with gr.Blocks(title="Advanced RAG Techniques", theme=gr.themes.Soft()) as demo: # type: ignore
-        
-        gr.Markdown("""
-        # 🚀 Advanced RAG Techniques Comparison Tool
-        """)
-        # This tool now includes **5 advanced retrieval techniques** alongside the original methods:
-        
-        # **🔥 New Advanced Techniques:**
-        # - **Multi-Query Retrieval** - Generate diverse queries for comprehensive results
-        # - **Parent-Child Retrieval** - Search with small chunks, return large context
-        # - **Contextual Compression** - AI-powered relevance filtering
-        # - **Cross-Encoder Reranking** - Superior relevance scoring
-        # - **Semantic Routing** - Smart query classification and routing
-        
-        # **Instructions:**
-        # 1. Enter a valid URL in the link box
-        # 2. Preview the webpage content 
-        # 3. Enter your question about the content
-        # 4. Select a RAG technique from the dropdown (try the new advanced ones!)
-        # 5. Click Submit to get your answer
-        # """)
-        
-        with gr.Row():
-            with gr.Column(scale=1):
-                # Input section
-                gr.Markdown("## 📝 Input Section")
-                
-                link_input = gr.Textbox(
-                    label="Website URL",
-                    placeholder="https://example.com/article",
-                    info="Enter the URL of the webpage you want to analyze"
-                )
-                
-                question_input = gr.Textbox(
-                    label="Your Question",
-                    placeholder="What is the main topic discussed in this article?",
-                    info="Ask any question about the content of the webpage"
-                )
-                
-                technique_dropdown = gr.Dropdown(
-                    choices=list(RAG_TECHNIQUES.keys()),
-                    label="RAG Technique",
-                    value="Multi-Query Retrieval",
-                    info="Choose the RAG technique - try the new advanced techniques!"
-                )
-                
-                submit_btn = gr.Button("🚀 Submit Query", variant="primary", size="lg")
-                
-                # Output section
-                gr.Markdown("## 💡 Answer")
-                answer_output = gr.Textbox(
-                    label="Generated Answer",
-                    lines=10,
-                    interactive=False,
-                    placeholder="Your answer will appear here..."
-                )
-            
-            with gr.Column(scale=1):
-                # Webpage preview section
-                gr.Markdown("## 🌐 Webpage Preview")
-                webpage_preview = gr.HTML(
-                    label="Webpage Content",
-                    value="<p style='text-align: center; color: #666; padding: 50px;'>Enter a URL to preview the webpage</p>"
-                )
-        
-        # Event handlers
-        link_input.change(
-            fn=create_webpage_preview,
-            inputs=[link_input],
-            outputs=[webpage_preview]
-        )
-        
-        submit_btn.click(
-            fn=process_rag_query,
-            inputs=[link_input, question_input, technique_dropdown],
-            outputs=[answer_output]
-        )
-        
-        # Add some example links and questions
-        # gr.Markdown("""
-        # ## 📚 Example Usage & Technique Comparison
-        
-        # **Sample URLs to try:**
-        # - `https://lilianweng.github.io/posts/2023-06-23-agent/` (AI Agents blog post)
-        # - `https://docs.python.org/3/tutorial/` (Python Tutorial)
-        # - `https://en.wikipedia.org/wiki/Machine_learning` (Machine Learning Wikipedia)
-        
-        # **Sample Questions:**
-        # - "What is task decomposition for LLM agents?"
-        # - "What are the main components of an AI agent?"
-        # - "How does retrieval-augmented generation work?"
-        
-        # **💡 Pro Tip:** Try the same question with different techniques to see how results vary!
-        # """)
-        
-        # # Add advanced technique descriptions
-        # with gr.Accordion("🔧 Advanced RAG Techniques Explained", open=False):
-        #     gr.Markdown("""
-        #     ## Original Techniques:
-        #     **HyDE:** Generates a hypothetical answer first, then uses it to retrieve relevant documents.
-            
-        #     **Query Decomposition:** Breaks down complex questions into simpler sub-questions that are answered sequentially.
-            
-        #     **Query Expansion:** Generates multiple variations of the original query to improve retrieval coverage.
-            
-        #     **RAG Fusion:** Creates multiple related queries and uses reciprocal rank fusion to combine results.
-            
-        #     **Step Back Query:** Transforms specific questions into more general ones to retrieve broader context.
-            
-        #     ## 🚀 Advanced Techniques:
-        #     **Multi-Query Retrieval:** Generates 4+ diverse query perspectives and merges results for comprehensive coverage.
-            
-        #     **Parent-Child Retrieval:** Uses small chunks for precise matching but returns larger parent chunks for better context.
-            
-        #     **Contextual Compression:** Uses LLM to compress retrieved chunks, keeping only information relevant to your question.
-            
-        #     **Cross-Encoder Reranking:** Uses specialized neural models to score and rerank documents for superior relevance.
-            
-        #     **Semantic Routing:** Automatically classifies your query type (factual, conceptual, comparative, analytical) and routes to the best retrieval strategy.
-        #     """)
-        
-        # # Installation requirements
-        # with gr.Accordion("📦 Additional Dependencies for Advanced Techniques", open=False):
-        #     gr.Markdown("""
-        #     To use the advanced retrieval techniques, install these additional packages:
-            
-        #     ```bash
-        #     pip install sentence-transformers scikit-learn
-        #     ```
-            
-        #     If you encounter errors with advanced techniques, make sure these dependencies are installed.
-        #     """)
-    
-    return demo
-
-# Launch the application
-if __name__ == "__main__":
-    # Check if required environment variables are set
-    if not os.getenv("OPENAI_API_KEY"):
-        print("Warning: OPENAI_API_KEY not found in environment variables.")
-        print("Please make sure to set your OpenAI API key in your .env file.")
-    
-    # Create and launch the interface
-    demo = create_interface()
-    demo.launch(
-        server_name="127.0.0.1",  # Local access
-        server_port=7860,         # Default Gradio port
-        share=False,              # Set to True if you want a public link
-        debug=True,               # Enable debug mode for development
-        inbrowser=True            # Automatically open in browser
+import gradio as gr
+import os
+import hashlib
+import json
+import pickle
+from datetime import datetime, timedelta
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Import your original RAG technique modules
+from Hyde import get_answer_using_hyde
+from QueryDecomposition import get_answer_using_query_decomposition
+from QueryExpansion import get_answer_using_query_expansion
+from RagFusion import get_answer_using_rag_fusion
+from StepBackQuery import get_answer
+
+# Import new advanced retrieval techniques
+from AdvancedRag import (
+    get_answer_using_multi_query,
+    get_answer_using_parent_child,
+    get_answer_using_contextual_compression,
+    get_answer_using_cross_encoder,
+    get_answer_using_semantic_routing
+)
+
+load_dotenv()
+
+# Cache configuration
+CACHE_DIR = Path("rag_cache")
+CACHE_DIR.mkdir(exist_ok=True)
+CACHE_EXPIRY_HOURS = 24  # Cache expires after 24 hours
+
+# Extended dictionary mapping technique names to their corresponding functions
+RAG_TECHNIQUES = {
+    # Original Techniques
+    "HyDE (Hypothetical Document Embeddings)": get_answer_using_hyde,
+    "Query Decomposition": get_answer_using_query_decomposition,
+    "Query Expansion": get_answer_using_query_expansion,
+    "RAG Fusion": get_answer_using_rag_fusion,
+    "Step Back Query": get_answer,
+    
+    # Advanced Retrieval Techniques
+    "Multi-Query Retrieval": get_answer_using_multi_query,
+    "Parent-Child Retrieval": get_answer_using_parent_child,
+    "Contextual Compression": get_answer_using_contextual_compression,
+    "Cross-Encoder Reranking": get_answer_using_cross_encoder,
+    "Semantic Routing": get_answer_using_semantic_routing,
+}
+
+def generate_cache_key(link, technique):
+    """
+    Generate a unique cache key based on link and technique
+    """
+    cache_string = f"{link}_{technique}"
+    return hashlib.md5(cache_string.encode()).hexdigest()
+
+def get_cache_file_path(cache_key):
+    """
+    Get the full path for a cache file
+    """
+    return CACHE_DIR / f"{cache_key}.pkl"
+
+def is_cache_valid(cache_file_path):
+    """
+    Check if cache file exists and is not expired
+    """
+    if not cache_file_path.exists():
+        return False
+    
+    # Check if cache is expired
+    file_time = datetime.fromtimestamp(cache_file_path.stat().st_mtime)
+    expiry_time = datetime.now() - timedelta(hours=CACHE_EXPIRY_HOURS)
+    
+    return file_time > expiry_time
+
+def save_to_cache(cache_key, data):
+    """
+    Save data to cache file
+    """
+    try:
+        cache_file_path = get_cache_file_path(cache_key)
+        cache_data = {
+            'data': data,
+            'timestamp': datetime.now().isoformat(),
+            'cache_key': cache_key
+        }
+        
+        with open(cache_file_path, 'wb') as f:
+            pickle.dump(cache_data, f)
+        
+        print(f"✅ Cached result for key: {cache_key}")
+        return True
+    except Exception as e:
+        print(f"❌ Failed to save cache: {e}")
+        return False
+
+def load_from_cache(cache_key):
+    """
+    Load data from cache file
+    """
+    try:
+        cache_file_path = get_cache_file_path(cache_key)
+        
+        if not is_cache_valid(cache_file_path):
+            return None
+        
+        with open(cache_file_path, 'rb') as f:
+            cache_data = pickle.load(f)
+        
+        print(f"🎯 Cache hit for key: {cache_key}")
+        return cache_data['data']
+    except Exception as e:
+        print(f"❌ Failed to load cache: {e}")
+        return None
+
+def clear_expired_cache():
+    """
+    Automatically clear expired cache files
+    """
+    try:
+        cache_files = list(CACHE_DIR.glob("*.pkl"))
+        expired_count = 0
+        
+        for cache_file in cache_files:
+            if not is_cache_valid(cache_file):
+                cache_file.unlink()
+                expired_count += 1
+        
+        if expired_count > 0:
+            print(f"🧹 Auto-cleared {expired_count} expired cache files")
+    except Exception as e:
+        print(f"❌ Failed to auto-clear expired cache: {e}")
+
+def process_rag_query(link, question, technique):
+    """
+    Process the RAG query using the selected technique with caching
+    """
+    try:
+        if not link or not question:
+            return "Please provide both a link and a question."
+        
+        if not link.startswith(('http://', 'https://')):
+            return "Please provide a valid URL starting with http:// or https://"
+        
+        # Auto-clear expired cache files
+        clear_expired_cache()
+        
+        # Generate cache key based on link and technique
+        cache_key = generate_cache_key(link, technique)
+        
+        # Try to load from cache first
+        cached_result = load_from_cache(cache_key)
+        if cached_result is not None:
+            # Check if we have this specific question cached
+            if isinstance(cached_result, dict) and question in cached_result:
+                return cached_result[question]
+        
+        # Get the corresponding function for the selected technique
+        rag_function = RAG_TECHNIQUES.get(technique)
+        
+        if not rag_function:
+            return "Invalid technique selected."
+        
+        print(f"🔄 Processing new query: {technique} for {link}")
+        
+        # Call the appropriate RAG function
+        answer = rag_function(link, question)
+        
+        # Save to cache
+        if cached_result is None:
+            cached_result = {}
+        elif not isinstance(cached_result, dict):
+            cached_result = {}
+        
+        cached_result[question] = answer
+        save_to_cache(cache_key, cached_result)
+        
+        return answer
+        
+    except Exception as e:
+        return f"Error processing query: {str(e)}\n\nNote: Advanced techniques require additional dependencies. Make sure you have installed: sentence-transformers, scikit-learn"
+
+def create_webpage_preview(link):
+    """
+    Create an HTML iframe to preview the webpage
+    """
+    if not link:
+        return ""
+    
+    if not link.startswith(('http://', 'https://')):
+        return "<p style='color: red;'>Please provide a valid URL starting with http:// or https://</p>"
+    
+    # Create an iframe to display the webpage
+    iframe_html = f"""
+    <div style="width: 100%; height: 500px; border: 1px solid #ccc; border-radius: 5px;">
+        <iframe src="{link}" width="100%" height="100%" frameborder="0" 
+                style="border-radius: 5px;">
+            <p>Your browser does not support iframes. 
+            <a href="{link}" target="_blank">Click here to open the link</a></p>
+        </iframe>
+    </div>
+    """
+    return iframe_html
+
+# Create the Gradio interface
+def create_interface():
+    with gr.Blocks(title="Advanced RAG Techniques", theme=gr.themes.Soft()) as demo: # type: ignore
+        
+        gr.Markdown("""
+        # 🚀 Advanced RAG Techniques Comparison Tool
+        """)
+        # This tool now includes **5 advanced retrieval techniques** alongside the original methods:
+        
+        # **🔥 New Advanced Techniques:**
+        # - **Multi-Query Retrieval** - Generate diverse queries for comprehensive results
+        # - **Parent-Child Retrieval** - Search with small chunks, return large context
+        # - **Contextual Compression** - AI-powered relevance filtering
+        # - **Cross-Encoder Reranking** - Superior relevance scoring
+        # - **Semantic Routing** - Smart query classification and routing
+        
+        # **Instructions:**
+        # 1. Enter a valid URL in the link box
+        # 2. Preview the webpage content 
+        # 3. Enter your question about the content
+        # 4. Select a RAG technique from the dropdown (try the new advanced ones!)
+        # 5. Click Submit to get your answer
+        # """)
+        
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Input section
+                gr.Markdown("## 📝 Input Section")
+                
+                link_input = gr.Textbox(
+                    label="Website URL",
+                    placeholder="https://example.com/article",
+                    info="Enter the URL of the webpage you want to analyze"
+                )
+                
+                question_input = gr.Textbox(
+                    label="Your Question",
+                    placeholder="What is the main topic discussed in this article?",
+                    info="Ask any question about the content of the webpage"
+                )
+                
+                technique_dropdown = gr.Dropdown(
+                    choices=list(RAG_TECHNIQUES.keys()),
+                    label="RAG Technique",
+                    value="Multi-Query Retrieval",
+                    info="Choose the RAG technique - try the new advanced techniques!"
+                )
+                
+                submit_btn = gr.Button("🚀 Submit Query", variant="primary", size="lg")
+                
+                # Output section
+                gr.Markdown("## 💡 Answer")
+                answer_output = gr.Textbox(
+                    label="Generated Answer",
+                    lines=10,
+                    interactive=False,
+                    placeholder="Your answer will appear here..."
+                )
+            
+            with gr.Column(scale=1):
+                # Webpage preview section
+                gr.Markdown("## 🌐 Webpage Preview")
+                webpage_preview = gr.HTML(
+                    label="Webpage Content",
+                    value="<p style='text-align: center; color: #666; padding: 50px;'>Enter a URL to preview the webpage</p>"
+                )
+        
+        # Event handlers
+        link_input.change(
+            fn=create_webpage_preview,
+            inputs=[link_input],
+            outputs=[webpage_preview]
+        )
+        
+        submit_btn.click(
+            fn=process_rag_query,
+            inputs=[link_input, question_input, technique_dropdown],
+            outputs=[answer_output]
+        )
+        
+        # Add some example links and questions
+        # gr.Markdown("""
+        # ## 📚 Example Usage & Technique Comparison
+        
+        # **Sample URLs to try:**
+        # - `https://lilianweng.github.io/posts/2023-06-23-agent/` (AI Agents blog post)
+        # - `https://docs.python.org/3/tutorial/` (Python Tutorial)
+        # - `https://en.wikipedia.org/wiki/Machine_learning` (Machine Learning Wikipedia)
+        
+        # **Sample Questions:**
+        # - "What is task decomposition for LLM agents?"
+        # - "What are the main components of an AI agent?"
+        # - "How does retrieval-augmented generation work?"
+        
+        # **💡 Pro Tip:** Try the same question with different techniques to see how results vary!
+        # """)
+        
+        # # Add advanced technique descriptions
+        # with gr.Accordion("🔧 Advanced RAG Techniques Explained", open=False):
+        #     gr.Markdown("""
+        #     ## Original Techniques:
+        #     **HyDE:** Generates a hypothetical answer first, then uses it to retrieve relevant documents.
+            
+        #     **Query Decomposition:** Breaks down complex questions into simpler sub-questions that are answered sequentially.
+            
+        #     **Query Expansion:** Generates multiple variations of the original query to improve retrieval coverage.
+            
+        #     **RAG Fusion:** Creates multiple related queries and uses reciprocal rank fusion to combine results.
+            
+        #     **Step Back Query:** Transforms specific questions into more general ones to retrieve broader context.
+            
+        #     ## 🚀 Advanced Techniques:
+        #     **Multi-Query Retrieval:** Generates 4+ diverse query perspectives and merges results for comprehensive coverage.
+            
+        #     **Parent-Child Retrieval:** Uses small chunks for precise matching but returns larger parent chunks for better context.
+            
+        #     **Contextual Compression:** Uses LLM to compress retrieved chunks, keeping only information relevant to your question.
+            
+        #     **Cross-Encoder Reranking:** Uses specialized neural models to score and rerank documents for superior relevance.
+            
+        #     **Semantic Routing:** Automatically classifies your query type (factual, conceptual, comparative, analytical) and routes to the best retrieval strategy.
+        #     """)
+        
+        # # Installation requirements
+        # with gr.Accordion("📦 Additional Dependencies for Advanced Techniques", open=False):
+        #     gr.Markdown("""
+        #     To use the advanced retrieval techniques, install these additional packages:
+            
+        #     ```bash
+        #     pip install sentence-transformers scikit-learn
+        #     ```
+            
+        #     If you encounter errors with advanced techniques, make sure these dependencies are installed.
+        #     """)
+    
+    return demo
+
+# Launch the application
+if __name__ == "__main__":
+    # Check if required environment variables are set
+    if not os.getenv("OPENAI_API_KEY"):
+        print("Warning: OPENAI_API_KEY not found in environment variables.")
+        print("Please make sure to set your OpenAI API key in your .env file.")
+    
+    # Create and launch the interface
+    demo = create_interface()
+    demo.launch(
+        share=True,              # Set to True if you want a public link
     )