Update app.py

app.py

@@ -13,6 +13,7 @@ from langchain.text_splitter import CharacterTextSplitter
 from langchain.chains import ConversationalRetrievalChain
 from langchain.memory import ConversationBufferMemory
 import urllib.parse
+import plotly.express as px
 
 # Ensure Playwright installs required browsers and dependencies
 subprocess.run(["playwright", "install"])
@@ -27,7 +28,6 @@ graph_config = {
     },
 }
 
-
 def get_data(url):
     smart_scraper_graph = SmartScraperGraph(
         prompt=(
@@ -40,15 +40,13 @@ def get_data(url):
     )
     return smart_scraper_graph.run()
 
-
 def process_multiple_urls(urls):
     """
-    Process multiple URLs with progress tracking
+    Process multiple URLs with progress tracking.
     """
     all_data = {"grants": []}
     progress_bar = st.progress(0)
     status_container = st.empty()
-
     total_urls = len(urls)
     for index, url in enumerate(urls):
         try:
@@ -56,11 +54,8 @@ def process_multiple_urls(urls):
             if not url:
                 continue
 
-            # Update progress
             progress = (index + 1) / total_urls
             progress_bar.progress(progress)
-
-            # Show current status
             status_container.markdown(
                 f"""
 **Processing URL {index+1} of {total_urls}**  
@@ -69,25 +64,20 @@ def process_multiple_urls(urls):
 ⏳ Remaining: {total_urls - index - 1}
 """
             )
-
-            # Scrape data
             result = get_data(url)
             if result and "grants" in result:
                 all_data["grants"].extend(result["grants"])
         except Exception as e:
             st.error(f"Error processing {url}: {str(e)}")
             continue
-
     progress_bar.empty()
     status_container.empty()
     return all_data
 
-
 def convert_to_csv(data):
     df = pd.DataFrame(data["grants"])
     return df.to_csv(index=False).encode("utf-8")
 
-
 def convert_to_excel(data):
     df = pd.DataFrame(data["grants"])
     buffer = io.BytesIO()
@@ -95,70 +85,76 @@ def convert_to_excel(data):
         df.to_excel(writer, sheet_name="Grants", index=False)
     return buffer.getvalue()
 
-
 def create_knowledge_base(data):
     documents = []
     for grant in data["grants"]:
         doc_parts = [f"{key.replace('_', ' ').title()}: {value}" for key, value in grant.items()]
         documents.append("\n".join(doc_parts))
-
     text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     texts = text_splitter.create_documents(documents)
-
     embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY)
     vectorstore = FAISS.from_documents(texts, embeddings)
-
     llm = ChatGoogleGenerativeAI(
         model="gemini-2.0-flash-thinking-exp", google_api_key=GOOGLE_API_KEY, temperature=0
     )
     memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
     return ConversationalRetrievalChain.from_llm(llm, vectorstore.as_retriever(), memory=memory)
 
-
 def get_shareable_link(file_data, file_name, file_type):
     b64 = base64.b64encode(file_data).decode()
     return f"data:{file_type};base64,{b64}"
 
-
-def main():
-    st.sidebar.title("Quantilytix Grant Scraper")
-    st.sidebar.image("logoqb.jpeg", use_container_width=True)
-
-    # Initialize session state for scraped data and chat history if not already present
-    if "scraped_data" not in st.session_state:
-        st.session_state.scraped_data = None
-    if "chat_history" not in st.session_state:
-        st.session_state.chat_history = []
-    if "chat_interface_active" not in st.session_state:
-        st.session_state.chat_interface_active = False
-
-    # Multi-URL input
-    url_input = st.sidebar.text_area(
+def display_dashboard(data):
+    df = pd.DataFrame(data["grants"])
+    if df.empty:
+        st.info("No data available for dashboard.")
+        return
+    st.subheader("Grants Dashboard")
+    # Filtering by sector if available
+    if "sector" in df.columns:
+        sectors = df["sector"].dropna().unique().tolist()
+        selected_sector = st.selectbox("Select Sector", options=["All"] + sectors)
+        if selected_sector != "All":
+            df = df[df["sector"] == selected_sector]
+    # Visualizations: Distribution of Grant Values
+    if "value" in df.columns:
+        df["value"] = pd.to_numeric(df["value"], errors="coerce")
+        fig_value = px.histogram(df, x="value", nbins=20, title="Distribution of Grant Values")
+        st.plotly_chart(fig_value)
+    # Visualization: Grants by Organisation
+    if "organisation" in df.columns:
+        fig_org = px.pie(df, names="organisation", title="Grants by Organisation")
+        st.plotly_chart(fig_org)
+    st.dataframe(df)
+
+def display_scrape_tab():
+    st.header("Scrape Grants")
+    url_input = st.text_area(
         "Enter URLs (one per line)",
         height=150,
         help="Enter multiple URLs separated by new lines",
     )
-
-    if st.sidebar.button("Get grants"):
+    if st.button("Start Scraping"):
         if url_input:
             urls = [url.strip() for url in url_input.split("\n") if url.strip()]
             if urls:
-                try:
-                    with st.spinner("Starting scraping process..."):
-                        result = process_multiple_urls(urls)
-                        st.session_state.scraped_data = result
-                        st.success(f"Scraped {len(result['grants'])} grants from {len(urls)} URLs!")
-                except Exception as e:
-                    st.error(f"Error in scraping process: {e}")
+                with st.spinner("Scraping grants..."):
+                    result = process_multiple_urls(urls)
+                    st.session_state.scraped_data = result
+                    st.success(f"Scraped {len(result['grants'])} grants from {len(urls)} URL(s)!")
             else:
                 st.warning("Please enter valid URLs.")
         else:
             st.warning("Please enter at least one URL.")
 
-    if st.session_state.scraped_data:
-        selected_format = st.sidebar.selectbox("Select Download Format", ("CSV", "Excel"))
+def display_download_tab():
+    st.header("Download & Explore Data")
+    if st.session_state.get("scraped_data"):
         result = st.session_state.scraped_data
-
+        df = pd.DataFrame(result["grants"])
+        st.subheader(f"Data Preview ({len(df)} grants)")
+        st.dataframe(df)
+        selected_format = st.selectbox("Select Download Format", ("CSV", "Excel"))
         if selected_format == "CSV":
             file_data = convert_to_csv(result)
             file_name = "grants.csv"
@@ -167,53 +163,92 @@ def main():
             file_data = convert_to_excel(result)
             file_name = "grants.xlsx"
             file_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-
         b64 = base64.b64encode(file_data).decode()
         download_link = f"<a href='data:{file_type};base64,{b64}' download='{file_name}'>Download {selected_format}</a>"
-        st.sidebar.markdown(download_link, unsafe_allow_html=True)
-
+        st.markdown(download_link, unsafe_allow_html=True)
         shareable_link = get_shareable_link(file_data, file_name, file_type)
-        st.sidebar.markdown("---")
-        st.sidebar.markdown("**Share Options:**")
-
+        st.markdown("---")
+        st.markdown("**Share Options:**")
         whatsapp_url = f"https://api.whatsapp.com/send?text={urllib.parse.quote(f'Check out this file: {shareable_link}')}"
-        st.sidebar.markdown(f"📱 [Share via WhatsApp]({whatsapp_url})")
-
+        st.markdown(f"📱 [Share via WhatsApp]({whatsapp_url})")
         email_subject = urllib.parse.quote("Check out this grants file")
         email_body = urllib.parse.quote(f"Download the file here: {shareable_link}")
         email_url = f"mailto:?subject={email_subject}&body={email_body}"
-        st.sidebar.markdown(f"📧 [Share via Email]({email_url})")
-
-        with st.expander(f"Preview of data ({len(result['grants'])} grants)"):
-            st.dataframe(result["grants"])
-
-        if st.sidebar.button("Load data as Knowledge Base"):
-            st.session_state.qa_chain = create_knowledge_base(result)
+        st.markdown(f"📧 [Share via Email]({email_url})")
+    else:
+        st.info("No scraped data available. Please scrape grants first.")
+
+def display_chat_tab():
+    st.header("Knowledge Base Chat")
+    st.info("Ask questions about the grants data.")
+    if st.session_state.get("scraped_data"):
+        if st.button("Load Data as Knowledge Base"):
+            st.session_state.qa_chain = create_knowledge_base(st.session_state.scraped_data)
             st.session_state.chat_interface_active = True
-            st.session_state.chat_history = []  # Initialize chat_history when KB is loaded
-
-    if st.session_state.get("chat_interface_active"):
-        st.header("Chat Interface Loaded. Start asking questions about the grants!")
-        st.image("logoqb.jpeg", width=150)
-        query = st.text_input("Ask a question about the grants:", key="chat_input")
-
-        if query:
-            if st.session_state.qa_chain:
+            st.session_state.chat_history = []
+            st.success("Knowledge base loaded!")
+        if st.session_state.get("chat_interface_active"):
+            query = st.text_input("Enter your query:")
+            if query:
                 response = st.session_state.qa_chain({"question": query})
                 st.session_state.chat_history.append({"query": query, "response": response["answer"]})
-            else:
-                st.error("Knowledge base not loaded. Please load the knowledge base first.")
-
-        for chat in st.session_state.get("chat_history", []):
-            st.markdown(
-                f"<p style='color: #8C92AC;'><strong>You:</strong> {chat['query']}</p>",
-                unsafe_allow_html=True,
-            )
-            st.markdown(
-                f"<p style='color: #6699CC;'><strong>Grants Bot:</strong> {chat['response']}</p>",
-                unsafe_allow_html=True,
-            )
+            for chat in st.session_state.get("chat_history", []):
+                st.markdown(f"**You:** {chat['query']}")
+                st.markdown(f"**Grants Bot:** {chat['response']}")
+        else:
+            st.info("Load the knowledge base to start chatting.")
+    else:
+        st.info("No scraped data available. Please scrape grants first.")
+
+def display_alerts_tab():
+    st.header("Automated Alerts Setup")
+    st.write("Configure your personalized alerts for new grant opportunities.")
+    keyword = st.text_input("Keyword for Alerts", help="E.g., 'AI', 'sustainable research'")
+    sector = st.text_input("Sector", help="E.g., 'health', 'technology'")
+    email = st.text_input("Your Email", help="Enter your email to receive alerts")
+    if st.button("Save Alert Preferences"):
+        # Placeholder for saving alert preferences—integrate with an email service in a full implementation.
+        st.success("Alert preferences saved! You will be notified when matching grants are found.")
+    st.info("Note: Automated alert functionality is under development and will be integrated soon.")
 
+def main():
+    st.set_page_config(page_title="Quantilytix Grants Platform", layout="wide", initial_sidebar_state="expanded")
+    # Custom CSS styling for a modern look
+    st.markdown("""
+    <style>
+    .main {
+        background-color: #f5f5f5;
+    }
+    .sidebar .sidebar-content {
+        background-image: linear-gradient(#2e7bcf, #2e7bcf);
+        color: white;
+    }
+    </style>
+    """, unsafe_allow_html=True)
+    
+    st.sidebar.title("Quantilytix Grants Platform")
+    st.sidebar.image("logoqb.jpeg", use_column_width=True)
+    app_mode = st.sidebar.radio("Navigation", ["Scrape Grants", "Download & Explore", "Dashboard", "Knowledge Base Chat", "Automated Alerts"])
+    
+    if app_mode == "Scrape Grants":
+        display_scrape_tab()
+    elif app_mode == "Download & Explore":
+        display_download_tab()
+    elif app_mode == "Dashboard":
+        if st.session_state.get("scraped_data"):
+            display_dashboard(st.session_state.scraped_data)
+        else:
+            st.info("No data available. Please scrape grants first.")
+    elif app_mode == "Knowledge Base Chat":
+        display_chat_tab()
+    elif app_mode == "Automated Alerts":
+        display_alerts_tab()
 
 if __name__ == "__main__":
-    main()
+    if "scraped_data" not in st.session_state:
+        st.session_state.scraped_data = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+    if "chat_interface_active" not in st.session_state:
+        st.session_state.chat_interface_active = False
+    main()