Adding auto-ingest

app.py

@@ -1,76 +1,209 @@
-"""Streamlit app for Abalone RAG chatbot."""
 import os
+
 os.environ.setdefault("LANGCHAIN_TELEMETRY_ENABLED", "false")
 os.environ.setdefault("LANGCHAIN_DISABLE_TELEMETRY", "true")
 os.environ.setdefault("CHROMA_TELEMETRY_ENABLED", "false")
 
 import streamlit as st
-
 from src.vectorstore import get_retriever
 from src.qa_chain import make_conversational_chain
-
+from src.ingest import ingest as run_ingest
 
 st.set_page_config(page_title="Abalone RAG Chatbot", page_icon="🐚")
 
 st.title("Abalone RAG Chatbot")
+st.write(
+    "Ask natural-language questions about abalone studies and data. "
+    "The app uses a local Chroma vectorstore and OpenAI to retrieve and answer."
+)
+
+st.sidebar.header("Configuration")
+
+model_name = st.sidebar.selectbox(
+    "Model",
+    options=["gpt-3.5-turbo", "gpt-4"],
+    index=0,
+)
+
+top_k = st.sidebar.slider(
+    "Number of retrieved chunks (k)",
+    min_value=2,
+    max_value=10,
+    value=4,
+)
+
+data_dir = st.sidebar.text_input("Data directory", value="./data")
+persist_dir = st.sidebar.text_input("Vectorstore directory", value="./vectorstore")
+
+chunk_size = st.sidebar.number_input(
+    "Chunk size",
+    min_value=200,
+    max_value=4000,
+    value=1000,
+    step=100,
+)
+
+chunk_overlap = st.sidebar.number_input(
+    "Chunk overlap",
+    min_value=0,
+    max_value=1000,
+    value=200,
+    step=50,
+)
+
+st.sidebar.markdown("---")
+st.sidebar.caption(
+    "If the vectorstore is missing or invalid, the app will attempt to ingest "
+    "the data automatically using these settings."
+)
 
 if "chat_history" not in st.session_state:
     st.session_state["chat_history"] = []
 
-with st.sidebar:
-    st.header("Settings")
-    model_name = st.selectbox("Model", ["gpt-3.5-turbo", "gpt-4"], index=0)
-    top_k = st.number_input("Retriever top_k", min_value=1, max_value=10, value=4)
-    if st.button("Rebuild vectorstore (ingest)"):
-        st.info("Rebuild requested. Run ingestion script or push data to trigger rebuild.")
+if "retriever_initialized" not in st.session_state:
+    st.session_state["retriever_initialized"] = False
 
-OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
-if not OPENAI_API_KEY:
-    st.error("OPENAI_API_KEY not found. Set the OPENAI_API_KEY environment variable or add it to Hugging Face Spaces Secrets.")
-    st.stop()
+def ensure_openai_key() -> bool:
+    if not os.environ.get("OPENAI_API_KEY"):
+        st.error("OPENAI_API_KEY is not set.")
+        return False
+    return True
+
+@st.cache_resource(show_spinner=False)
+def build_or_load_retriever_cached(
+        data_dir: str,
+        persist_dir: str,
+        top_k: int,
+        chunk_size: int,
+        chunk_overlap: int,
+):
+    try:
+        return get_retriever(persist_dir=persist_dir, top_k=top_k)
+    except Exception:
+        run_ingest(
+            data_dir=data_dir,
+            persist_dir=persist_dir,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+        return get_retriever(persist_dir=persist_dir, top_k=top_k)
+
+def get_or_build_retriever_with_ui():
+    if not ensure_openai_key():
+        return None
+    try:
+        return build_or_load_retriever_cached(
+            data_dir=data_dir,
+            persist_dir=persist_dir,
+            top_k=top_k,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+        )
+    except Exception as e:
+        st.error(
+            "Could not initialize vectorstore.\n\n"
+            f"Details: `{e}`"
+        )
+        return None
 
-persist_dir = "./vectorstore"
 retriever = None
-try:
-    retriever = get_retriever(persist_dir=persist_dir, top_k=top_k)
-except Exception as e:
-    st.warning("Vectorstore not found or not initialized. Please run the ingestion script to build it.\n" + str(e))
-
-
-if retriever:
-    chain = make_conversational_chain(retriever, model_name=model_name)
-
-    user_input = st.text_input("Ask a question about Abalone", key="input")
-    if st.button("Send") and user_input:
-        with st.spinner("Thinking..."):
-            prior_history = [(h.get("question"), h.get("answer", "")) for h in st.session_state.get("chat_history", [])]
-            result = chain({"question": user_input, "chat_history": prior_history})
-            answer = result.get("answer") or result.get("output_text") or ""
-            source_docs = result.get("source_documents") or []
-        st.session_state.setdefault("chat_history", [])
-        st.session_state["chat_history"].append({"question": user_input, "answer": answer, "sources": source_docs})
-
-    if st.session_state.get("chat_history"):
-        for item in reversed(st.session_state.get("chat_history", [])):
-            st.markdown(f"**User:** {item.get('question')}")
-            st.markdown(f"**Assistant:** {item.get('answer')}")
-            sources = item.get("sources") or []
-            if sources:
-                with st.expander("Sources"):
-                    for sd in sources:
-                        if isinstance(sd, dict):
-                            meta = sd.get("metadata", {})
-                            content_preview = sd.get("page_content") or sd.get("content") or sd.get("text", "")
-                        else:
-                            meta = getattr(sd, "metadata", {}) or {}
-                            content_preview = getattr(sd, "page_content", None)
-                            if content_preview is None:
-                                content_preview = getattr(sd, "content", "")
+with st.spinner("Initializing vectorstore and retriever..."):
+    retriever = get_or_build_retriever_with_ui()
+
+if retriever is None:
+    st.info("No retriever available. Fix the errors above and refresh the page.")
+    st.stop()
+
+st.success("Vectorstore and retriever are ready.")
+
+chain = make_conversational_chain(retriever, model_name=model_name)
+
+if st.session_state["chat_history"]:
+    st.subheader("Conversation")
+    for i, turn in enumerate(st.session_state["chat_history"]):
+        st.markdown(f"**You:** {turn['question']}")
+        st.markdown(f"**Abalone Bot:** {turn['answer']}")
+        if turn.get("sources"):
+            with st.expander(f"Show sources for question {i + 1}"):
+                for j, src in enumerate(turn["sources"], start=1):
+                    st.markdown(f"**Source {j}:**")
+                    meta = src.get("metadata", {})
+                    if meta:
                         st.write(meta)
-                        if content_preview:
-                            try:
-                                st.write(content_preview[:400])
-                            except Exception:
-                                st.write(str(content_preview))
-else:
-    st.info("No retriever available. Ingest data first.")
+                    preview = src.get("content_preview", "")
+                    if preview:
+                        st.write(preview)
+
+st.subheader("Ask a question")
+
+user_input = st.text_input(
+    "Ask a question about abalone (biology, data, methodology, etc.)",
+    key="user_question_input",
+)
+
+send_clicked = st.button("Send")
+
+if send_clicked and user_input:
+    if not ensure_openai_key():
+        st.stop()
+
+    with st.spinner("Thinking..."):
+        prior_history = [
+            (h.get("question"), h.get("answer", "")) for h in st.session_state["chat_history"]
+        ]
+
+        result = chain(
+            {"question": user_input, "chat_history": prior_history}
+        )
+
+        answer = (
+                result.get("answer")
+                or result.get("result")
+                or result.get("output_text")
+                or ""
+        )
+        source_docs = result.get("source_documents") or []
+
+        sources_for_ui = []
+        for sd in source_docs:
+            if isinstance(sd, dict):
+                meta = sd.get("metadata", {}) or {}
+                content_preview = sd.get("page_content") or sd.get("content") or sd.get("text", "")
+            else:
+                meta = getattr(sd, "metadata", {}) or {}
+                content_preview = getattr(sd, "page_content", None)
+                if content_preview is None:
+                    content_preview = getattr(sd, "content", "")
+            if content_preview is None:
+                content_preview = ""
+            sources_for_ui.append(
+                {
+                    "metadata": meta,
+                    "content_preview": str(content_preview)[:500],
+                }
+            )
+
+    st.session_state["chat_history"].append(
+        {
+            "question": user_input,
+            "answer": answer,
+            "sources": sources_for_ui,
+        }
+    )
+
+    st.markdown("---")
+    st.markdown("### Latest Answer")
+    st.markdown(f"**You:** {user_input}")
+    st.markdown(f"**Abalone Bot:** {answer}")
+
+    if sources_for_ui:
+        with st.expander("Show sources for this answer"):
+            for i, src in enumerate(sources_for_ui, start=1):
+                st.markdown(f"**Source {i}:**")
+                if src["metadata"]:
+                    st.write(src["metadata"])
+                if src["content_preview"]:
+                    st.write(src["content_preview"])
+
+elif send_clicked and not user_input:
+    st.warning("Please enter a question before clicking Send.")