Upload 5 files

agent_workflow.py

@@ -0,0 +1,317 @@
+import os
+import logging
+from typing import List, Dict, Any, Optional, Tuple, Callable, Union
+from dotenv import load_dotenv
+from llm_providers import LLMProvider
+from langchain.schema import HumanMessage
+from tantivy_search_agent import TantivySearchAgent
+
+load_dotenv()
+
+class SearchAgent:
+    def __init__(self, tantivy_agent: TantivySearchAgent, provider_name: str = "Claude"):
+        """Initialize the search agent with Tantivy agent and LLM client"""
+        self.tantivy_agent = tantivy_agent
+        self.logger = logging.getLogger(__name__)
+        
+        # Initialize LLM provider
+        self.llm_provider = LLMProvider()
+        self.llm = None
+        self.set_provider(provider_name)
+        
+        self.min_confidence_threshold = 0.5
+
+    def set_provider(self, provider_name: str) -> None:
+        self.llm = self.llm_provider.get_provider(provider_name)
+        if not self.llm:
+            raise ValueError(f"Provider {provider_name} not available")
+        self.current_provider = provider_name
+
+    def get_available_providers(self) -> list[str]:
+        return self.llm_provider.get_available_providers()
+
+    def get_query(self, query: str, failed_queries: List[Dict[str, str]] = []) -> str:
+        """Generate a Tantivy query using Claude, considering previously failed queries"""
+        try:
+            if not self.llm:
+                raise ValueError("LLM provider not initialized")
+
+            prompt = (
+                "Create a query for this search request with the following restrictions:\n"+
+                self.tantivy_agent.get_query_instructions()+                
+                "\n\nAdditional instructions: \n"
+                "1. return only the search query without any other text\n"
+                "2. Use only Hebrew terms for the search query\n"
+                "3. the corpus to search in is an ancient Hebrew corpus - Tora and Talmud. so Try to use ancient Hebrew terms and or Talmudic expressions."
+                "4. prevent modern words that are not common in talmudic texts \n"              
+                f"the search request: {query}"
+            )
+            
+            if failed_queries:
+                prompt += (
+                    f"\n\nPrevious failed queries:\n"+
+                    "------------------------\n"+                    
+                    '\n'.join(f"Query: {q['query']}, Reason: {q['reason']}" for q in failed_queries)+
+                    "\n\n"
+                    "Please generate an alternative query that:\n"
+                    "1. Uses different Hebrew synonyms or related terms\n"
+                    "2. Tries broader or more general terms\n"
+                    "3. Adjusts proximity values or uses wildcards\n"
+                    "4. Prevents using modern words that are not common in ancient hebrew and talmud texts\n"
+                )
+            
+            response = self.llm.invoke([HumanMessage(content=prompt)])
+            tantivy_query = response.content.strip()  
+            self.logger.info(f"Generated Tantivy query: {tantivy_query}")
+            return tantivy_query
+            
+        except Exception as e:
+            self.logger.error(f"Error generating query: {e}")
+            # Fallback to basic quoted search
+            return f'"{query}"'
+
+    def _evaluate_results(self, results: List[Dict[str, Any]], query: str) -> Dict[str, Any]:
+        """Evaluate search results using Claude with confidence scoring"""
+        if not self.llm:
+            raise ValueError("LLM provider not initialized")
+     
+        # Prepare context from results
+        context = "\n".join(f"Result {i}. Source: {r.get('reference',[])}\n Text: {r.get('text', [])}"
+            for i, r in enumerate(results)
+                )
+
+        try:
+            message = self.llm.invoke([HumanMessage(content=f"""Evaluate the search results for answering this question:
+                    Question: {query}
+                    
+                    Search Results:
+                    {context}
+                    
+                    Provide evaluation in this format (3 lines):
+                    Confidence score (0.0 to 1.0) indicating how well the results can answer the question. this line should include only the number return, don't include '[line 1]'
+                    ACCEPT if score >= {self.min_confidence_threshold}, REFINE if score < {self.min_confidence_threshold}. return only the word ACCEPT or REFINE.
+                    Detailed explanation of what information is present or missing, don't include '[line 3]'. it should be only in Hebrew
+                             """)])
+            lines = message.content.strip().replace('\n\n', '\n').split('\n')
+            confidence = float(lines[0])
+            decision = lines[1].upper()
+            explanation = lines[2]
+            
+            is_good = decision == 'ACCEPT'
+
+            self.logger.info(f"Evaluation: Confidence={confidence}, Decision={decision}")
+            self.logger.info(f"Explanation: {explanation}")
+            
+            return {
+                "confidence": confidence,
+                "is_sufficient": is_good,
+                "explanation": explanation,
+             
+            }
+            
+        except Exception as e:
+            self.logger.error(f"Error evaluating results: {e}")
+            # Fallback to simple evaluation
+            return {
+                "confidence": 0.0,
+                "is_sufficient": False,
+                "explanation": "",         
+            }
+
+    def _generate_answer(self, query: str, results: List[Dict[str, Any]]) -> str:
+        """Generate answer using Claude with improved context utilization"""
+        if not self.llm:
+            raise ValueError("LLM provider not initialized")
+
+        if not results:
+            return "לא נמצאו תוצאות"
+
+        # Prepare context from results
+        context = "\n".join(f"Result {i+1}. Source: {r.get('reference',[])}\n Text: {r.get('text', [])}"
+            for i, r in enumerate(results)
+                )
+        
+        try:
+            message = self.llm.invoke([HumanMessage(content=f"""Based on these search results, answer this question:
+                    Question: {query}
+                    
+                    Search Results:
+                    {context}
+                    
+                    Requirements for your answer:
+                    1. Use only information from the search results
+                    2. Be comprehensive but concise
+                    3. Structure the answer clearly
+                    4. If any aspect of the question cannot be fully answered, acknowledge this
+                    5. cite sources for each fact or information you use
+                    6. The answer should be only in Hebrew
+                    """)])
+            return message.content.strip()
+            
+        except Exception as e:
+            self.logger.error(f"Error generating answer: {e}")
+            return f"I encountered an error generating the answer: {str(e)}"
+
+    def search_and_answer(self, query: str, num_results: int = 10, max_iterations: int = 3, 
+                         on_step: Optional[Callable[[Dict[str, Any]], None]] = None) -> Dict[str, Any]:
+        """Execute multi-step search process using Tantivy with streaming updates"""
+        steps = []
+        all_results = []
+        
+        # Step 1: Generate Tantivy query
+        initial_query = self.get_query(query)
+        step = {
+            'action': 'יצירת שאילתת חיפוש',
+            'description': 'נוצרה שאילתת חיפוש עבור מנוע החיפוש',
+            'results': [{'type': 'query', 'content': initial_query}]
+        }
+        steps.append(step)
+        if on_step:
+            on_step(step)
+        
+        # Step 2: Initial search with Tantivy query
+        results = self.tantivy_agent.search(initial_query, num_results)
+        
+        step = {
+            'action': 'חיפוש במאגר',
+            'description': f'חיפוש במאגר עבור שאילתת חיפוש: {initial_query}',
+            'results': [{'type': 'document', 'content': {
+                'title': r['title'],
+                'reference': r['reference'],
+                'topics': r['topics'],
+                'highlights': r['highlights'],
+                'score': r['score']
+            }} for r in results]
+        }
+        steps.append(step)
+        if on_step:
+            on_step(step)
+
+        failed_queries = []
+
+        if results.__len__() == 0:
+            failed_queries.append({'query': initial_query, 'reason': 'no results'})
+            is_sufficient = False
+        else:         
+            all_results.extend(results)
+            
+            # Step 3: Evaluate results
+            evaluation = self._evaluate_results(results, query)
+            confidence = evaluation['confidence']
+            is_sufficient = evaluation['is_sufficient']
+            explanation = evaluation['explanation']
+            
+            step = {
+                'action': 'דירוג תוצאות',
+                'description': 'דירוג תוצאות חיפוש',
+                'results': [{
+                    'type': 'evaluation',
+                    'content': {
+                        'status': 'accepted' if is_sufficient else 'insufficient',
+                        'confidence': confidence,
+                        'explanation': explanation,
+                    }
+                }]
+            }
+            steps.append(step)
+            if on_step:
+                on_step(step)
+
+            if not is_sufficient:
+                failed_queries.append({'query': initial_query, 'reason': explanation})
+            
+        # Step 4: Additional searches if needed
+        attempt = 2
+        while not is_sufficient and attempt < max_iterations:
+            # Generate new query
+            new_query = self.get_query(query, failed_queries)
+           
+            step = {
+                'action': f'יצירת שאילתה מחדש (ניסיון {attempt})',
+                'description': 'נוצרה שאילתת חיפוש נוספת עבור מנוע החיפוש',
+                'results': [
+                    {'type': 'new_query', 'content': new_query}
+                ]
+            }
+            steps.append(step)
+            if on_step:
+                on_step(step)
+            
+            # Search with new query
+            results = self.tantivy_agent.search(new_query, num_results)
+            
+            step = {
+                'action': f'חיפוש נוסף (ניסיון {attempt}) ',
+                'description': f'מחפש במאגר עבור שאילתת חיפוש: {new_query}',
+                'results': [{'type': 'document', 'content': {
+                    'title': r['title'],
+                    'reference': r['reference'],
+                    'topics': r['topics'],
+                    'highlights': r['highlights'],
+                    'score': r['score']
+                }} for r in results]
+            }
+            steps.append(step)
+            if on_step:
+                on_step(step)
+
+            if results.__len__() == 0:
+                failed_queries.append({'query': new_query, 'reason': 'no results'})
+            
+            else:
+                all_results.extend(results)
+            
+                # Re-evaluate with current results
+                evaluation = self._evaluate_results(results, query)
+                confidence = evaluation['confidence']
+                is_sufficient = evaluation['is_sufficient']
+                explanation = evaluation['explanation']
+                
+                step = {
+                    'action': f'דירוג תוצאות (ניסיון {attempt})',
+                    'description': 'דירוג תוצאות חיפוש לניסיון זה',
+                    'explanation': explanation,
+                    'results': [{
+                        'type': 'evaluation',
+                        'content': {
+                            'status': 'accepted' if is_sufficient else 'insufficient',
+                            'confidence': confidence,
+                            'explanation': explanation,
+                        }
+                    }]
+                }
+                steps.append(step)
+                if on_step:
+                    on_step(step)
+                
+                if not is_sufficient:
+                    failed_queries.append({'query': new_query, 'reason': explanation})
+            
+            attempt += 1
+        
+        # Step 5: Generate final answer
+        answer = self._generate_answer(query, all_results)
+        
+        final_result = {
+            'steps': steps,
+            'answer': answer,
+            'sources': [{
+                'title': r['title'],
+                'reference': r['reference'],
+                'topics': r['topics'],
+                'path': r['file_path'],
+                'highlights': r['highlights'],
+                'text': r['text'],
+                'score': r['score']
+            } for r in all_results]
+        }
+
+        # Send final result through callback
+        if on_step:
+            on_step({
+                'action': 'סיום',
+                'description': 'החיפוש הושלם',
+                'final_result': final_result
+            })
+
+        return final_result

llm_providers.py

@@ -0,0 +1,133 @@
+from langchain_anthropic import ChatAnthropic
+from langchain_openai import ChatOpenAI
+from langchain_ollama import ChatOllama
+from langchain_core.language_models.base import BaseLanguageModel
+from langchain_core.messages import BaseMessage, HumanMessage, AIMessage
+from typing import Optional, Dict, List, Any
+import os
+import requests
+import json
+from dotenv import load_dotenv
+from dataclasses import dataclass
+
+
+load_dotenv()
+
+
+@dataclass
+class GeminiResponse:
+    content: str
+
+
+class GeminiProvider:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+        self.base_url = "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent"
+        
+    def chat(self, messages: List[Dict[str, Any]]) -> GeminiResponse:
+        # Convert messages to Gemini format
+        gemini_messages = []
+        for msg in messages:
+            # Handle both dict and LangChain message objects
+            if isinstance(msg, BaseMessage):
+                role = "user" if isinstance(msg, HumanMessage) else "model"
+                content = msg.content
+            else:
+                role = "user" if msg["role"] == "human" else "model"
+                content = msg["content"]
+                
+            gemini_messages.append({
+                "role": role,
+                "parts": [{"text": content}]
+            })
+        
+        # Prepare the request
+        headers = {
+            "Content-Type": "application/json"
+        }
+        
+        params = {
+            "key": self.api_key
+        }
+        
+        data = {
+            "contents": gemini_messages,
+            "generationConfig": {
+                "temperature": 0.7,
+                "topP": 0.8,
+                "topK": 40,
+                "maxOutputTokens": 2048,
+            }
+        }
+        
+        try:
+            response = requests.post(
+                self.base_url,
+                headers=headers,
+                params=params,
+                json=data,
+                verify='C:\\ProgramData\\NetFree\\CA\\netfree-ca-bundle-curl.crt'
+            )
+            response.raise_for_status()
+            
+            result = response.json()
+            if "candidates" in result and len(result["candidates"]) > 0:
+                return GeminiResponse(content=result["candidates"][0]["content"]["parts"][0]["text"])
+            else:
+                raise Exception("No response generated")
+                
+        except Exception as e:
+            raise Exception(f"Error calling Gemini API: {str(e)}")
+
+    def invoke(self, messages: List[BaseMessage], **kwargs) -> GeminiResponse:
+        return self.chat(messages)
+
+    def generate(self, prompts, **kwargs) -> GeminiResponse:
+        if isinstance(prompts, str):
+            return self.invoke([HumanMessage(content=prompts)])
+        elif isinstance(prompts, list):
+            return self.invoke([HumanMessage(content=prompts[0])])
+        raise ValueError("Unsupported prompt format")
+
+class LLMProvider:
+    def __init__(self):
+        self.providers: Dict[str, Any] = {}
+        self._setup_providers()
+
+    def _setup_providers(self):
+        os.environ['REQUESTS_CA_BUNDLE'] = 'C:\\ProgramData\\NetFree\\CA\\netfree-ca-bundle-curl.crt'
+        
+           # Google Gemini
+        if google_key := os.getenv('GOOGLE_API_KEY'):
+            self.providers['Gemini'] = GeminiProvider(api_key=google_key)
+
+        
+        # Anthropic
+        if anthropic_key := os.getenv('ANTHROPIC_API_KEY'):
+            self.providers['Claude'] = ChatAnthropic(
+                api_key=anthropic_key,
+                model_name="claude-3-5-sonnet-20241022",
+                
+            )
+
+        # OpenAI
+        if openai_key := os.getenv('OPENAI_API_KEY'):
+            self.providers['ChatGPT'] = ChatOpenAI(
+                api_key=openai_key,
+                model_name="gpt-4o-2024-11-20"
+            )
+
+     
+        # Ollama (local)
+        try:
+            self.providers['Ollama-dictalm2.0'] = ChatOllama(model="dictaLM")
+        except Exception:
+            pass  # Ollama not available
+
+    def get_available_providers(self) -> list[str]:
+        """Return list of available provider names"""
+        return list(self.providers.keys())
+
+    def get_provider(self, name: str) -> Optional[Any]:
+        """Get LLM provider by name"""
+        return self.providers.get(name)

requirements.txt

@@ -0,0 +1,13 @@
+langchain
+python-dotenv
+flet
+langchain-community
+langchain-core
+langchain-openai
+langchain-anthropic
+langchain-ollama
+ollama
+requests
+tantivy
+streamlit
+gdown

streamlit_app.py

@@ -0,0 +1,275 @@
+import streamlit as st
+from tantivy_search_agent import TantivySearchAgent
+from agent_workflow import SearchAgent
+import os
+from typing import Optional, List
+from dotenv import load_dotenv
+import gdown
+
+# Load environment variables
+load_dotenv()
+
+class SearchAgentUI:
+    def __init__(self):
+        self.tantivy_agent: Optional[TantivySearchAgent] = None
+        self.agent: Optional[SearchAgent] = None
+        self.index_path ="./index" # os.getenv("INDEX_PATH", "./index")
+        # Google Drive folder ID for the index
+        self.gdrive_index_id = os.getenv("GDRIVE_INDEX_ID", "1lpbBCPimwcNfC0VZOlQueA4SHNGIp5_t")
+
+    def download_index_from_gdrive(self) -> bool:
+        """Download index folder from Google Drive"""
+        try:
+            # Create a temporary zip file path
+            zip_path = "index.zip"
+            # Download the folder as a zip file
+            url = f"https://drive.google.com/uc?id={self.gdrive_index_id}"
+            
+            # Create a progress bar and status text
+            progress_text = st.empty()
+            progress_bar = st.progress(0)
+            
+            def progress_callback(progress):
+                progress_bar.progress(progress)
+                progress_text.text(f"מוריד... {progress:.1f}%")
+            
+            # Download with progress callback
+            gdown.download(url, zip_path, quiet=False, callback=progress_callback)
+            
+            # Update status for extraction
+            progress_text.text("מחלץ קבצים...")
+            progress_bar.progress(100)
+            
+            # Extract the zip file
+            import zipfile
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                zip_ref.extractall(".")
+            
+            # Remove the zip file
+            os.remove(zip_path)
+            
+            # Clear the progress indicators
+            progress_text.empty()
+            progress_bar.empty()
+            
+            return True
+        except Exception as e:
+            st.error(f"Failed to download index: {str(e)}")
+            return False
+
+    def get_available_providers(self) -> List[str]:
+        """Get available providers without creating a SearchAgent instance"""
+        temp_tantivy = TantivySearchAgent(self.index_path)
+        temp_agent = SearchAgent(temp_tantivy)
+        return temp_agent.get_available_providers()
+
+    def initialize_system(self):
+        try:
+            # Check if index folder exists
+            if not os.path.exists(self.index_path):
+                st.warning("Index folder not found. Attempting to download from Google Drive...")
+                if not self.download_index_from_gdrive():
+                    return False, "שגיאה: לא ניתן להוריד את האינדקס", []
+                st.success("Index downloaded successfully!")
+
+            self.tantivy_agent = TantivySearchAgent(self.index_path)
+            if self.tantivy_agent.validate_index():
+                available_providers = self.get_available_providers()
+                self.agent = SearchAgent(
+                    self.tantivy_agent,
+                    provider_name=st.session_state.get('provider', available_providers[0])
+                )
+                return True, "המערכת מוכנה לחיפוש", available_providers
+            else:
+                return False, "שגיאה: אינדקס לא תקין", []
+        except Exception as ex:
+            return False, f"שגיאה באתחול המערכת: {str(ex)}", []
+
+    def main(self):
+        st.set_page_config(
+            page_title="איתוריא",
+            layout="wide",
+            initial_sidebar_state="collapsed"
+        )
+
+        # Enhanced RTL support and styling
+        st.markdown("""
+        <style>
+            .stApp {
+                direction: rtl;
+            }
+            .stTextInput > div > div > input {
+                direction: rtl;
+            }
+            .stSelectbox > div > div > div {
+                direction: rtl;
+            }
+            .stNumberInput > div > div > input {
+                direction: rtl;
+            }
+            .search-step {
+                border: 1px solid #e0e0e0;
+                border-radius: 5px;
+                padding: 10px;
+                margin: 5px 0;
+                background-color: #f8f9fa;
+            }
+            .document-group {
+                border: 1px solid #e3f2fd;
+                border-radius: 5px;
+                padding: 10px;
+                margin: 5px 0;
+                background-color: #f5f9ff;
+            }
+            .document-item {
+                border: 1px solid #e0e0e0;
+                border-radius: 5px;
+                padding: 10px;
+                margin: 5px 0;
+                background-color: white;
+            }
+        </style>
+        """, unsafe_allow_html=True)
+
+        # Initialize system
+        success, status_msg, available_providers = self.initialize_system()
+
+        # Header layout
+        col1, col2, col3 = st.columns([2,1,1])
+
+        with col1:
+            if success:
+                st.success(status_msg)
+            else:
+                st.error(status_msg)
+
+        with col2:
+            if 'provider' not in st.session_state:
+                st.session_state.provider = available_providers[0] if available_providers else None
+            
+            if available_providers:
+                provider = st.selectbox(
+                    "ספק בינה מלאכותית",
+                    options=available_providers,
+                    key='provider'
+                )
+                if self.agent:
+                    self.agent.set_provider(provider)
+
+        with col3:
+            col3_1, col3_2 = st.columns(2)
+            with col3_1:
+                max_iterations = st.number_input(
+                    "מספר נסיונות מקסימלי",
+                    min_value=1,
+                    value=3,
+                    key='max_iterations'
+                )
+            with col3_2:
+                results_per_search = st.number_input(
+                    "תוצאות לכל חיפוש",
+                    min_value=1,
+                    value=5,
+                    key='results_per_search'
+                )
+
+        # Search input
+        query = st.text_input(
+            "הכנס שאילתת חיפוש",
+            disabled=not success,
+            placeholder="הקלד את שאילתת החיפוש שלך כאן...",
+            key='search_query'
+        )
+
+        # Search button
+        if (st.button('חפש', disabled=not success) or query) and query!="" and self.agent:          
+                try:
+                    if 'steps' not in st.session_state:
+                        st.session_state.steps = []
+                    
+                    steps_container = st.container()
+                    answer_container = st.container()
+                    sources_container = st.container()
+
+                    with steps_container:
+                        st.subheader("צעדי תהליך החיפוש")
+
+                    def handle_step_update(step):
+                        if 'final_result' in step:
+                            final_result = step['final_result']
+                            
+                            with answer_container:
+                                st.subheader("תשובה סופית")
+                                st.info(final_result['answer'])
+
+                            if final_result['sources']:
+                                with sources_container:
+                                    st.subheader("מסמכי מקור")
+                                    st.markdown(f"נמצאו {len(final_result['sources'])} תוצאות")
+                                    
+                                    for i, source in enumerate(final_result['sources']):
+                                        with st.expander(f"תוצאה {i+1}: {source['reference']} (ציון: {source['score']:.2f})"):
+                                            st.write(source['text'])
+ 
+                        else:
+                            with steps_container:
+                                step_number = len(st.session_state.steps) + 1
+                                st.markdown(f"""
+                                <div class='search-step'>
+                                    <strong>צעד {step_number}. {step['action']}</strong>
+                                </div>
+                                """, unsafe_allow_html=True)
+                                st.markdown(f"**{step['description']}**")
+                                
+                                if 'results' in step:
+                                    documents = []
+                                    
+                                    for r in step['results']:
+                                        if r['type'] == 'query':
+                                            st.markdown("**שאילתת חיפוש:**")
+                                            st.code(r['content'])
+                                        
+                                        elif r['type'] == 'document':
+                                            documents.append(r['content'])
+                                        
+                                        elif r['type'] == 'evaluation':
+                                            content = r['content']
+                                            status = "✓" if content['status'] == 'accepted' else "↻"
+                                            confidence = f"ביטחון: {content['confidence']}"
+                                            if content['status'] == 'accepted':
+                                                st.success(f"{status} {confidence}")
+                                            else:
+                                                st.warning(f"{status} {confidence}")
+                                            if content['explanation']:
+                                                st.info(content['explanation'])
+                                        
+                                        elif r['type'] == 'new_query':
+                                            st.markdown("**ניסיון הבא:**")
+                                            st.code(r['content'])
+                                    
+                                    # Display documents if any were found
+                                    if documents:
+                                        for i, doc in enumerate(documents):
+                                            with st.expander(f"{doc['reference']} (ציון: {doc['score']:.2f})"):
+                                                st.write(doc['highlights'][0])
+                                    
+                                st.markdown("---")
+                            st.session_state.steps.append(step)
+
+                    # Clear previous steps before starting new search
+                    st.session_state.steps = []
+                    
+                    # Start the search process
+                    self.agent.search_and_answer(
+                        query=query,
+                        num_results=results_per_search,
+                        max_iterations=max_iterations,
+                        on_step=handle_step_update
+                    )
+
+                except Exception as ex:
+                    st.error(f"שגיאת חיפוש: {str(ex)}")
+
+if __name__ == "__main__":
+    app = SearchAgentUI()
+    app.main()