Upload 11 files

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.replit

@@ -0,0 +1,55 @@
+# .replit
+# Specifies the command to run the app
+run = "streamlit run app.py"
+
+# The main entry point file
+entrypoint = "app.py"
+modules = ["python-3.11"]
+
+# Tells Replit's package manager about dependencies
+[packager]
+language = "python3"
+
+# Optional: Ignore specific packages if needed, though usually handled by requirements.txt
+# ignoredPackages = []
+
+# Optional: Improves Python development experience in Replit
+[languages.python]
+pattern = "**/*.py"
+[languages.python.languageServer]
+name = "pyright"
+
+[deployment]
+deploymentTarget = "cloudrun"
+run = ["sh", "-c", "streamlit run --server.address 0.0.0.0 --server.headless true --server.enableCORS=false --server.enableWebsocketCompression=false app.py"]
+build = ["sh", "-c", "pip install --upgrade pip"]
+
+[nix]
+
+[[ports]]
+localPort = 8501
+externalPort = 80
+
+[[ports]]
+localPort = 8502
+externalPort = 3000
+
+[[ports]]
+localPort = 8503
+externalPort = 3001
+
+[workflows]
+runButton = "Run"
+
+[[workflows.workflow]]
+name = "Run"
+author = 22737092
+mode = "sequential"
+
+[[workflows.workflow.tasks]]
+task = "shell.exec"
+args = "streamlit run app.py"
+
+# Environment variables can also be set here, but Secrets are recommended for API keys
+# [env]
+# LANGSMITH_TRACING = "true"

app.py

@@ -0,0 +1,232 @@
+import streamlit as st
+import time
+import asyncio
+import nest_asyncio
+import traceback
+from typing import List, Dict, Any
+import re  # for extracting citation IDs
+
+# --- Configuration and Service Initialization ---
+try:
+    print("App: Loading config...")
+    import config
+    print("App: Loading utils...")
+    from utils import clean_source_text
+    print("App: Loading services...")
+    from services.retriever import init_retriever, get_retriever_status
+    from services.openai_service import init_openai_client, get_openai_status
+    print("App: Loading RAG processor...")
+    from rag_processor import execute_validate_generate_pipeline, PIPELINE_VALIDATE_GENERATE_GPT4O
+    print("App: Imports successful.")
+except ImportError as e:
+    st.error(f"Fatal Error: Module import failed. {e}", icon="🚨")
+    traceback.print_exc()
+    st.stop()
+except Exception as e:
+    st.error(f"Fatal Error during initial setup: {e}", icon="🚨")
+    traceback.print_exc()
+    st.stop()
+
+nest_asyncio.apply()
+
+# --- Initialize Required Services ---
+print("App: Initializing services...")
+try:
+    retriever_ready_init, retriever_msg_init = init_retriever()
+    openai_ready_init, openai_msg_init = init_openai_client()
+    print("App: Service initialization calls complete.")
+except Exception as init_err:
+    st.error(f"Error during service initialization: {init_err}", icon="🔥")
+    traceback.print_exc()
+
+# --- Streamlit Page Configuration and Styling ---
+st.set_page_config(page_title="Divrey Yoel AI Chat (GPT-4o Gen)", layout="wide")
+st.markdown("""<style> /* ... Keep existing styles ... */ </style>""", unsafe_allow_html=True)
+st.markdown("<h1 class='rtl-text'> דברות קודש - חיפוש ועיון</h1>", unsafe_allow_html=True)
+st.markdown("<p class='rtl-text'>מבוסס על ספרי דברי יואל מסאטמאר זצוק'ל זי'ע - אחזור מידע חכם (RAG)</p>", unsafe_allow_html=True)
+st.markdown("<p class='rtl-text' style='font-size: 0.9em; color: #555;'>תהליך: אחזור -> אימות (GPT-4o) -> יצירה (GPT-4o)</p>", unsafe_allow_html=True)
+
+# --- UI Helper Functions ---
+def display_sidebar() -> Dict[str, Any]:
+    st.sidebar.markdown("<h3 class='rtl-text'>מצב המערכת</h3>", unsafe_allow_html=True)
+    retriever_ready, _ = get_retriever_status()
+    openai_ready, _ = get_openai_status()
+    st.sidebar.markdown(
+        f"<p class='rtl-text'><strong>מאחזר (Pinecone):</strong> {'✅' if retriever_ready else '❌'}</p>",
+        unsafe_allow_html=True
+    )
+    if not retriever_ready:
+        st.sidebar.error("מאחזר אינו זמין.", icon="🛑")
+        st.stop()
+    st.sidebar.markdown("<hr>", unsafe_allow_html=True)
+    st.sidebar.markdown(
+        f"<p class='rtl-text'><strong>OpenAI ({config.OPENAI_VALIDATION_MODEL} / {config.OPENAI_GENERATION_MODEL}):</strong> {'✅' if openai_ready else '❌'}</p>",
+        unsafe_allow_html=True
+    )
+    if not openai_ready:
+        st.sidebar.error("OpenAI אינו זמין.", icon="⚠️")
+    st.sidebar.markdown("<hr>", unsafe_allow_html=True)
+    st.sidebar.markdown("<h3 class='rtl-text'>הגדרות חיפוש</h3>", unsafe_allow_html=True)
+    n_retrieve = st.sidebar.slider("מספר פסקאות לאחזור", 1, 300, config.DEFAULT_N_RETRIEVE)
+    max_validate = min(n_retrieve, 100)
+    n_validate = st.sidebar.slider(
+        "פסקאות לאימות (GPT-4o)",
+        1,
+        max_validate,
+        min(config.DEFAULT_N_VALIDATE, max_validate),
+        disabled=not openai_ready
+    )
+    st.sidebar.info("התשובות מבוססות רק על המקורות שאומתו.", icon="ℹ️")
+    return {"n_retrieve": n_retrieve, "n_validate": n_validate, "services_ready": (retriever_ready and openai_ready)}
+
+
+def display_chat_message(message: Dict[str, Any]):
+    role = message.get("role", "assistant")
+    with st.chat_message(role):
+        st.markdown(message.get('content', ''), unsafe_allow_html=True)
+        if role == "assistant" and message.get("final_docs"):
+            docs = message["final_docs"]
+            exp_title = f"<span class='rtl-text'>הצג {len(docs)} קטעי מקור שנשלחו למחולל (GPT-4o)</span>"
+            with st.expander(exp_title, expanded=False):
+                st.markdown("<div dir='rtl' class='expander-content'>", unsafe_allow_html=True)
+                for i, doc in enumerate(docs, start=1):
+                    if not isinstance(doc, dict):
+                        continue
+                    source = doc.get('source_name', '') or 'מקור לא ידוע'
+                    text = clean_source_text(doc.get('hebrew_text', ''))
+                    st.markdown(
+                        f"<div class='source-info rtl-text'><strong>מקור {i}:</strong> {source}</div>",
+                        unsafe_allow_html=True
+                    )
+                    st.markdown(f"<div class='hebrew-text'>{text}</div>", unsafe_allow_html=True)
+                st.markdown("</div>", unsafe_allow_html=True)
+
+
+def display_status_updates(status_log: List[str]):
+    if status_log:
+        with st.expander("<span class='rtl-text'>הצג פרטי עיבוד</span>", expanded=False):
+            for u in status_log:
+                st.markdown(
+                    f"<code class='status-update rtl-text'>- {u}</code>",
+                    unsafe_allow_html=True
+                )
+
+# --- Main Application Logic ---
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+rag_params = display_sidebar()
+
+# Render history
+for msg in st.session_state.messages:
+    display_chat_message(msg)
+
+if prompt := st.chat_input("שאל שאלה בענייני חסידות...", disabled=not rag_params["services_ready"]):
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    display_chat_message(st.session_state.messages[-1])
+
+    with st.chat_message("assistant"):
+        msg_placeholder = st.empty()
+        status_container = st.status("מעבד בקשה...", expanded=True)
+        chunks: List[str] = []
+        try:
+            def status_cb(m):
+                status_container.update(label=f"<span class='rtl-text'>{m}</span>")
+            def stream_cb(c):
+                chunks.append(c)
+                msg_placeholder.markdown(
+                    f"<div dir='rtl' class='rtl-text'>{''.join(chunks)}▌</div>",
+                    unsafe_allow_html=True
+                )
+
+            loop = asyncio.get_event_loop()
+            final_rag = loop.run_until_complete(
+                execute_validate_generate_pipeline(
+                    history=st.session_state.messages,
+                    params=rag_params,
+                    status_callback=status_cb,
+                    stream_callback=stream_cb
+                )
+            )
+
+            if isinstance(final_rag, dict):
+                raw = final_rag.get("final_response", "")
+                err = final_rag.get("error")
+                log = final_rag.get("status_log", [])
+                docs = final_rag.get("generator_input_documents", [])
+                pipeline = final_rag.get("pipeline_used", PIPELINE_VALIDATE_GENERATE_GPT4O)
+
+                # wrap in RTL div if needed
+                final = raw
+                if not (err and final.strip().startswith("<div")) and not final.strip().startswith((
+                    '<div', '<p', '<ul', '<ol', '<strong'
+                )):
+                    final = f"<div dir='rtl' class='rtl-text'>{final or 'לא התקבלה תשובה מהמחולל.'}</div>"
+                msg_placeholder.markdown(final, unsafe_allow_html=True)
+
+                # --- Show only cited paragraphs ---
+                cited_ids = set(re.findall(r'\(מקור\s*([0-9]+)\)', raw))
+                if cited_ids:
+                    enumerated_docs = list(enumerate(docs, start=1))
+                    docs_to_show = [(idx, doc) for idx, doc in enumerated_docs if str(idx) in cited_ids]
+                else:
+                    docs_to_show = list(enumerate(docs, start=1))
+
+                if docs_to_show:
+                    label = f"<span class='rtl-text'>הצג {len(docs_to_show)} קטעי מקור שהוזכרו בתשובה</span>"
+                    with st.expander(label, expanded=False):
+                        st.markdown("<div dir='rtl' class='expander-content'>", unsafe_allow_html=True)
+                        for idx, doc in docs_to_show:
+                            source = doc.get('source_name', '') or 'מקור לא ידוע'
+                            text = clean_source_text(doc.get('hebrew_text', ''))
+                            st.markdown(
+                                f"<div class='source-info rtl-text'><strong>מקור {idx}:</strong> {source}</div>",
+                                unsafe_allow_html=True
+                            )
+                            st.markdown(f"<div class='hebrew-text'>{text}</div>", unsafe_allow_html=True)
+                        st.markdown("</div>", unsafe_allow_html=True)
+                # --- end filter display ---
+
+                # store assistant message
+                assistant_data = {
+                    "role": "assistant",
+                    "content": final,
+                    "final_docs": docs,
+                    "pipeline_used": pipeline,
+                    "status_log": log,
+                    "error": err
+                }
+                st.session_state.messages.append(assistant_data)
+                display_status_updates(log)
+                if err:
+                    status_container.update(label="שגיאה בעיבוד!", state="error", expanded=False)
+                else:
+                    status_container.update(label="העיבוד הושלם!", state="complete", expanded=False)
+            else:
+                msg_placeholder.markdown(
+                    "<div dir='rtl' class='rtl-text'><strong>שגיאה בלתי צפויה בתקשורת.</strong></div>",
+                    unsafe_allow_html=True
+                )
+                st.session_state.messages.append({
+                    "role": "assistant",
+                    "content": "שגיאה בלתי צפויה בתקשורת.",
+                    "final_docs": [],
+                    "pipeline_used": "Error",
+                    "status_log": ["Unexpected result"],
+                    "error": "Unexpected"
+                })
+                status_container.update(label="שגיאה בלתי צפויה!", state="error", expanded=False)
+        except Exception as e:
+            traceback.print_exc()
+            err_html = (f"<div dir='rtl' class='rtl-text'><strong>שגיאה קריטית!</strong><br>נסה לרענן."  
+                         f"<details><summary>פרטים</summary><pre>{traceback.format_exc()}</pre></details></div>")
+            msg_placeholder.error(err_html, icon="🔥")
+            st.session_state.messages.append({
+                "role": "assistant",
+                "content": err_html,
+                "final_docs": [],
+                "pipeline_used": "Critical Error",
+                "status_log": [f"Critical: {type(e).__name__}"],
+                "error": str(e)
+            })
+            status_container.update(label=str(e), state="error", expanded=False)

config.py

@@ -0,0 +1,79 @@
+# config.py (Updated for OpenAI Generation)
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# --- LangSmith Configuration ---
+LANGSMITH_ENDPOINT = os.environ.get("LANGSMITH_ENDPOINT", "https://api.smith.langchain.com")
+LANGSMITH_TRACING = os.environ.get("LANGSMITH_TRACING", "true")
+LANGSMITH_API_KEY = os.environ.get("LANGSMITH_API_KEY")
+LANGSMITH_PROJECT = os.environ.get("LANGSMITH_PROJECT", "DivreyYoel-RAG-GPT4-Gen") # Updated project name
+
+# --- API Keys (Required) ---
+# ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY") # No longer needed
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") # Used for Embedding, Validation, AND Generation
+PINECONE_API_KEY = os.environ.get("PINECONE_API_KEY")
+
+# --- Model Configuration ---
+# Embedding Model (Retriever)
+EMBEDDING_MODEL = os.environ.get("OPENAI_EMBEDDING_MODEL", "text-embedding-3-large")
+# Validation Model (OpenAI)
+OPENAI_VALIDATION_MODEL = os.environ.get("OPENAI_VALIDATION_MODEL", "gpt-4o")
+# Generation Model (OpenAI) - Can be the same or different from validation
+OPENAI_GENERATION_MODEL = os.environ.get("OPENAI_GENERATION_MODEL", "gpt-4o") # Using GPT-4o for generation too
+
+# --- Pinecone Configuration ---
+PINECONE_INDEX_NAME = os.environ.get("PINECONE_INDEX_NAME", "chassidus-index")
+
+# --- Default RAG Pipeline Parameters ---
+DEFAULT_N_RETRIEVE = 100
+DEFAULT_N_VALIDATE = 50
+
+# --- System Prompts ---
+# --- REMOVED ANTHROPIC_SYSTEM_PROMPT ---
+
+# --- NEW OpenAI System Prompt ---
+OPENAI_SYSTEM_PROMPT = """You are an expert assistant specializing in Chassidic texts, particularly the works of the Satmar Rebbe, Rabbi Yoel Teitelbaum (Divrei Yoel).
+Your task is to answer the user's question based *exclusively* on the provided source text snippets (paragraphs from relevant books). Do not use any prior knowledge or external information.
+
+**Source Text Format:**
+The relevant source texts will be provided below under the heading "Source Texts:". Each source is numbered and includes an ID.
+
+**Response Requirements:**
+1.  **Language:** Respond **exclusively in Hebrew**.
+2.  **Basis:** Base your answer *strictly* on the information contained within the provided "Source Texts:". Do not infer, add external knowledge, or answer if the context does not contain relevant information.
+3.  **Attribution (Optional but Recommended):** When possible, mention the source number (e.g., "כפי שמופיע במקור 3") where the information comes from. Do not invent information. Use quotes sparingly and only when essential, quoting the Hebrew text directly.
+4.  **Completeness:** Synthesize information from *multiple* relevant sources if they contribute to the answer.
+5.  **Handling Lack of Information:** If the provided sources do not contain information relevant to the question, state clearly in Hebrew that the provided texts do not contain the answer (e.g., "על פי המקורות שסופקו, אין מידע לענות על שאלה זו."). Do not attempt to answer based on outside knowledge.
+6.  **Clarity and Conciseness:** Provide a clear, well-structured, and concise answer in Hebrew. Focus on directly answering the user's question.
+7.  **Tone:** Maintain a formal and respectful tone appropriate for discussing religious texts.
+8.  **No Greetings/Closings:** Do not include introductory greetings (e.g., "שלום") or concluding remarks (e.g., "בברכה", "מקווה שעזרתי"). Focus solely on the answer.
+"""
+
+# --- Helper Functions ---
+def check_env_vars():
+    """Checks if essential API keys are present."""
+    missing_keys = []
+    if not LANGSMITH_API_KEY: missing_keys.append("LANGSMITH_API_KEY")
+    # if not ANTHROPIC_API_KEY: missing_keys.append("ANTHROPIC_API_KEY") # Removed check
+    if not OPENAI_API_KEY: missing_keys.append("OPENAI_API_KEY")
+    if not PINECONE_API_KEY: missing_keys.append("PINECONE_API_KEY")
+    return missing_keys
+
+def configure_langsmith():
+    """Sets LangSmith environment variables."""
+    os.environ["LANGSMITH_ENDPOINT"] = LANGSMITH_ENDPOINT
+    os.environ["LANGSMITH_TRACING"] = LANGSMITH_TRACING
+    if LANGSMITH_API_KEY: os.environ["LANGSMITH_API_KEY"] = LANGSMITH_API_KEY
+    if LANGSMITH_PROJECT: os.environ["LANGSMITH_PROJECT"] = LANGSMITH_PROJECT
+    print(f"LangSmith configured: Endpoint={LANGSMITH_ENDPOINT}, Tracing={LANGSMITH_TRACING}, Project={LANGSMITH_PROJECT or 'Default'}")
+
+# --- Initial Check on Import ---
+missing = check_env_vars()
+if missing:
+    print(f"Warning: Missing essential API keys in Replit Secrets: {', '.join(missing)}")
+else:
+    print("All essential API keys found in environment/secrets.")
+
+configure_langsmith()

pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "python-template"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+requires-python = ">=3.11"
+dependencies = []

rag_processor.py

@@ -0,0 +1,152 @@
+# rag_processor.py (Fixed Syntax Error AGAIN)
+import time
+import asyncio
+import traceback
+from typing import List, Dict, Any, Optional, Callable, Tuple
+from langsmith import traceable
+
+try:
+    import config
+    from services import retriever, openai_service
+except ImportError:
+    print("Error: Failed to import config or services in rag_processor.py")
+    raise SystemExit("Failed imports in rag_processor.py")
+
+PIPELINE_VALIDATE_GENERATE_GPT4O = "GPT-4o Validator + GPT-4o Synthesizer"
+StatusCallback = Callable[[str], None]
+
+# --- Step Functions ---
+
+@traceable(name="rag-step-retrieve")
+async def run_retrieval_step(query: str, n_retrieve: int, update_status: StatusCallback) -> List[Dict]:
+    update_status(f"1. מאחזר עד {n_retrieve} פסקאות מ-Pinecone...")
+    start_time = time.time(); retrieved_docs = retriever.retrieve_documents(query_text=query, n_results=n_retrieve)
+    retrieval_time = time.time() - start_time; status_msg = f"אוחזרו {len(retrieved_docs)} פסקאות ב-{retrieval_time:.2f} שניות."
+    update_status(f"1. {status_msg}")
+    if not retrieved_docs: update_status("1. לא אותרו מסמכים.")
+    return retrieved_docs
+
+@traceable(name="rag-step-gpt4o-filter")
+async def run_gpt4o_validation_filter_step(
+    docs_to_process: List[Dict], query: str, n_validate: int, update_status: StatusCallback
+) -> List[Dict]:
+    if not docs_to_process: update_status("2. [GPT-4o] דילוג על אימות - אין פסקאות."); return []
+    validation_count = min(len(docs_to_process), n_validate)
+    update_status(f"2. [GPT-4o] מתחיל אימות מקבילי ({validation_count} / {len(docs_to_process)} פסקאות)...")
+    validation_start_time = time.time()
+    tasks = [openai_service.validate_relevance_openai(doc, query, i) for i, doc in enumerate(docs_to_process[:validation_count])]
+    validation_results = await asyncio.gather(*tasks, return_exceptions=True)
+    passed_docs = []; passed_count, failed_validation_count, error_count = 0, 0, 0
+    update_status("3. [GPT-4o] סינון פסקאות לפי תוצאות אימות...")
+    for i, res in enumerate(validation_results):
+        original_doc = docs_to_process[i]
+        if isinstance(res, Exception): print(f"GPT-4o Validation Exception doc {i}: {res}"); error_count += 1
+        elif isinstance(res, dict) and 'validation' in res and 'paragraph_data' in res:
+            if res["validation"].get("contains_relevant_info") is True:
+                original_doc['validation_result'] = res["validation"]; passed_docs.append(original_doc); passed_count += 1
+            else: failed_validation_count += 1
+        else: print(f"GPT-4o Validation Unexpected result doc {i}: {type(res)}"); error_count += 1
+    validation_time = time.time() - validation_start_time
+    status_msg_val = f"אימות GPT-4o הושלם ({passed_count} עברו, {failed_validation_count} נדחו, {error_count} שגיאות) ב-{validation_time:.2f} שניות."
+    update_status(f"2. {status_msg_val}")
+    status_msg_filter = f"נאספו {len(passed_docs)} פסקאות רלוונטיות לאחר אימות GPT-4o."
+    update_status(f"3. {status_msg_filter}")
+    return passed_docs # Returns full docs that passed
+
+@traceable(name="rag-step-openai-generate")
+async def run_openai_generation_step(
+    history: List[Dict], context_documents: List[Dict], update_status: StatusCallback, stream_callback: Callable[[str], None]
+) -> Tuple[str, Optional[str]]:
+    generator_name = "OpenAI";
+    if not context_documents:
+        update_status(f"4. [{generator_name}] דילוג על יצירה - אין פסקאות להקשר.")
+        return "לא סופקו פסקאות רלוונטיות ליצירת התשובה.", None
+    update_status(f"4. [{generator_name}] מחולל תשובה סופית מ-{len(context_documents)} קטעי הקשר...")
+    start_gen_time = time.time()
+    try:
+        full_response = []; error_msg = None
+        generator = openai_service.generate_openai_stream(messages=history, context_documents=context_documents)
+        async for chunk in generator:
+            if isinstance(chunk, str) and chunk.strip().startswith("--- Error:"):
+                if not error_msg: error_msg = chunk.strip()
+                print(f"OpenAI stream yielded error: {chunk.strip()}"); break
+            elif isinstance(chunk, str): full_response.append(chunk); stream_callback(chunk)
+        final_response_text = "".join(full_response); gen_time = time.time() - start_gen_time
+        if error_msg: update_status(f"4. שגיאה ביצירת התשובה ({generator_name}) ב-{gen_time:.2f} שניות."); return final_response_text, error_msg
+        else: update_status(f"4. יצירת התשובה ({generator_name}) הושלמה ב-{gen_time:.2f} שניות."); return final_response_text, None
+    except Exception as gen_err:
+        gen_time = time.time() - start_gen_time; error_msg_critical = f"--- Error: Critical failure during {generator_name} generation ({type(gen_err).__name__}): {gen_err} ---"
+        update_status(f"4. שגיאה קריטית ביצירת התשובה ({generator_name}) ב-{gen_time:.2f} שניות."); traceback.print_exc(); return "", error_msg_critical
+
+
+# --- Main Pipeline Orchestrator ---
+@traceable(name="rag-execute-validate-generate-gpt4o-pipeline")
+async def execute_validate_generate_pipeline(
+    history: List[Dict], params: Dict[str, Any], status_callback: StatusCallback, stream_callback: Callable[[str], None]
+) -> Dict[str, Any]:
+    """
+    Orchestrates Retrieve -> Validate (GPT-4o) -> Generate (GPT-4o) pipeline.
+    Stores both full validated docs and simplified docs for generator input.
+    """
+    result = { "final_response": "", "validated_documents_full": [], "generator_input_documents": [], "status_log": [], "error": None, "pipeline_used": PIPELINE_VALIDATE_GENERATE_GPT4O }
+
+    # --- Corrected Initialization ---
+    status_log_internal = [] # Initialize the list on its own line
+
+    # Define the helper function on the next line, correctly indented
+    def update_status_and_log(message: str):
+        print(f"Status Update: {message}") # Log status to console
+        status_log_internal.append(message)
+        status_callback(message) # Update UI
+    # ------------------------------
+
+    current_query_text = ""
+    if history and isinstance(history, list):
+         for msg_ in reversed(history):
+             if isinstance(msg_, dict) and msg_.get("role") == "user": current_query_text = str(msg_.get("content") or ""); break
+    if not current_query_text: print("Error: Could not extract query."); result["error"] = "לא זוהתה שאלה."; result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"; result["status_log"] = status_log_internal; return result
+
+    try:
+        # --- 1. Retrieval ---
+        retrieved_docs = await run_retrieval_step(current_query_text, params['n_retrieve'], update_status_and_log)
+        if not retrieved_docs: result["error"] = "לא אותרו מקורות."; result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"; result["status_log"] = status_log_internal; return result
+
+        # --- 2. Validation ---
+        validated_docs_full = await run_gpt4o_validation_filter_step(retrieved_docs, current_query_text, params['n_validate'], update_status_and_log)
+        result["validated_documents_full"] = validated_docs_full # Store full docs for trace/debug
+        if not validated_docs_full: result["error"] = "לא נמצאו פסקאות רלוונטיות."; result["final_response"] = f"<div class='rtl-text'>{result['error']}</div>"; result["status_log"] = status_log_internal; update_status_and_log(f"4. {result['error']} לא ניתן להמשיך."); return result
+
+        # --- Simplify Docs for Generation ---
+        simplified_docs_for_generation = []
+        print(f"Processor: Simplifying {len(validated_docs_full)} docs...");
+        for doc in validated_docs_full:
+            if isinstance(doc, dict):
+                hebrew_text = doc.get('hebrew_text', '')
+                if hebrew_text:
+                    simplified_doc = {'hebrew_text': hebrew_text, 'original_id': doc.get('original_id', 'unknown'), 'source_name': doc.get('source_name', '')}
+                    if not simplified_doc['source_name']: del simplified_doc['source_name']
+                    simplified_docs_for_generation.append(simplified_doc)
+            else: print(f"Warn: Skipping non-dict item: {doc}")
+        result["generator_input_documents"] = simplified_docs_for_generation # Store simplified for UI
+        print(f"Processor: Created {len(simplified_docs_for_generation)} simplified docs.")
+
+        # --- 3. Generation ---
+        final_response_text, generation_error = await run_openai_generation_step(
+            history=history, context_documents=simplified_docs_for_generation, # Pass simplified list
+            update_status=update_status_and_log, stream_callback=stream_callback
+        )
+        result["final_response"] = final_response_text; result["error"] = generation_error
+
+        # Handle display of potential errors
+        if generation_error and not result["final_response"].strip().startswith(("<div", "לא סופקו")):
+             result["final_response"] = f"<div class='rtl-text'><strong>שגיאה ביצירת התשובה.</strong><br>פרטים: {generation_error}<br>---<br>{result['final_response']}</div>"
+        elif result["final_response"] == "לא סופקו פסקאות רלוונטיות ליצירת התשובה.":
+             result["final_response"] = f"<div class='rtl-text'>{result['final_response']}</div>"
+
+    except Exception as e: # General catch-all
+        error_type = type(e).__name__; error_msg = f"שגיאה קריטית RAG ({error_type}): {e}"; print(f"Critical RAG Error: {error_msg}"); traceback.print_exc()
+        result["error"] = error_msg; result["final_response"] = f"<div class='rtl-text'><strong>שגיאה קריטית! ({error_type})</strong><br>נסה שוב.<details><summary>פרטים</summary><pre>{traceback.format_exc()}</pre></details></div>"
+        update_status_and_log(f"שגיאה קריטית: {error_type}")
+
+    result["status_log"] = status_log_internal
+    return result

requirements.txt

@@ -0,0 +1,8 @@
+
+streamlit
+python-dotenv
+pinecone
+openai
+anthropic
+langsmith
+nest_asyncio

utils.py

@@ -0,0 +1,93 @@
+# utils.py (Updated for OpenAI context formatting)
+import re
+import os
+import time
+import traceback
+import openai
+from typing import Optional, List, Dict
+
+try:
+    import config
+except ImportError:
+    print("Error: config.py not found. Cannot proceed.")
+    raise SystemExit("config.py not found")
+
+# ... (keep openai_client init, clean_source_text, get_embedding) ...
+openai_client = None
+if config.OPENAI_API_KEY:
+    try:
+        openai_client = openai.OpenAI(api_key=config.OPENAI_API_KEY)
+        print("Utils: OpenAI client initialized for embeddings.")
+    except Exception as e:
+        print(f"Utils: Error initializing OpenAI client for embeddings: {e}")
+else:
+    print("Utils: Warning - OPENAI_API_KEY not found. Embeddings will fail.")
+
+def clean_source_text(text: Optional[str]) -> str:
+    if not text: return ""
+    text = text.replace('\x00', '').replace('\ufffd', '')
+    text = re.sub(r'\s+', ' ', text).strip()
+    return text
+
+def get_embedding(text: str, model: str = config.EMBEDDING_MODEL, max_retries: int = 3) -> Optional[List[float]]:
+    global openai_client
+    if not openai_client:
+        print("Error: OpenAI client not initialized (utils.py). Cannot get embedding.")
+        return None
+    if not text or not isinstance(text, str):
+        print("Error: Invalid input text for embedding.")
+        return None
+    cleaned_text = text.replace("\n", " ").strip()
+    if not cleaned_text:
+         print("Warning: Text is empty after cleaning, cannot get embedding.")
+         return None
+    attempt = 0
+    while attempt < max_retries:
+        try:
+            response = openai_client.embeddings.create(input=[cleaned_text], model=model)
+            return response.data[0].embedding
+        except openai.RateLimitError as e:
+            wait_time = (2 ** attempt); print(f"Rate limit embedding. Retrying in {wait_time}s..."); time.sleep(wait_time)
+            attempt += 1
+        except openai.APIConnectionError as e:
+             print(f"Connection error embedding. Retrying..."); time.sleep(2)
+             attempt += 1
+        except Exception as e:
+            print(f"Error generating embedding (Attempt {attempt + 1}/{max_retries}): {type(e).__name__}")
+            attempt += 1
+    print(f"Failed embedding after {max_retries} attempts.")
+    return None
+
+# --- REMOVED format_context_for_anthropic ---
+
+# --- NEW Function to format context for OpenAI ---
+def format_context_for_openai(documents: List[Dict]) -> str:
+    """Formats documents for the OpenAI prompt context section using numbered list."""
+    if not documents:
+        return "No source texts provided."
+    formatted_docs = []
+    language_key = 'hebrew_text'
+    id_key = 'original_id'
+    source_key = 'source_name' # Optional: Include source name if available
+
+    for index, doc in enumerate(documents):
+        if not isinstance(doc, dict):
+            print(f"Warning: Skipping non-dict item in documents list: {doc}")
+            continue
+
+        text = clean_source_text(doc.get(language_key, ''))
+        doc_id = doc.get(id_key, f'unknown_{index+1}')
+        source_name = doc.get(source_key, '') # Get source name
+
+        if text:
+            # Start with 1-based indexing for readability
+            header = f"Source {index + 1} (ID: {doc_id}"
+            if source_name:
+                header += f", SourceName: {source_name}"
+            header += ")"
+            formatted_docs.append(f"{header}:\n{text}\n---") # Add separator
+
+    if not formatted_docs:
+         return "No valid source texts could be formatted."
+
+    return "\n".join(formatted_docs)

uv.lock

@@ -0,0 +1,7 @@
+version = 1
+requires-python = ">=3.11"
+
+[[package]]
+name = "python-template"
+version = "0.1.0"
+source = { virtual = "." }