Update app.py

app.py

@@ -13,7 +13,6 @@ from langchain.text_splitter import CharacterTextSplitter
 from langchain.chains import ConversationalRetrievalChain
 from langchain.memory import ConversationBufferMemory
 import urllib.parse
-import plotly.express as px
 
 # Ensure Playwright installs required browsers and dependencies
 subprocess.run(["playwright", "install"])
@@ -28,6 +27,7 @@ graph_config = {
     },
 }
 
+
 def get_data(url):
     smart_scraper_graph = SmartScraperGraph(
         prompt=(
@@ -40,14 +40,16 @@ def get_data(url):
     )
     return smart_scraper_graph.run()
 
+
 def process_multiple_urls(urls):
     """
-    Process multiple URLs with progress tracking.
+    Process multiple URLs with enhanced progress tracking and user feedback.
     """
     all_data = {"grants": []}
     progress_bar = st.progress(0)
     status_container = st.empty()
     total_urls = len(urls)
+
     for index, url in enumerate(urls):
         try:
             url = url.strip()
@@ -58,26 +60,31 @@ def process_multiple_urls(urls):
             progress_bar.progress(progress)
             status_container.markdown(
                 f"""
-**Processing URL {index+1} of {total_urls}**  
-🔍 Scanning: `{url}`  
-✅ Completed: {index}/{total_urls}  
-⏳ Remaining: {total_urls - index - 1}
-"""
+    **Processing Grant Opportunities** 🚀
+    Scanning URL {index+1} of {total_urls}:  `{url}`
+    <br>
+    <p style='font-size: 0.9em; color: #6699CC;'>Completed: {index}/{total_urls} | Remaining: {total_urls - index - 1}</p>
+    """,
+                unsafe_allow_html=True,
             )
+
             result = get_data(url)
             if result and "grants" in result:
                 all_data["grants"].extend(result["grants"])
         except Exception as e:
-            st.error(f"Error processing {url}: {str(e)}")
+            st.error(f"⚠️ Error processing URL: {url} - {str(e)}")
             continue
+
     progress_bar.empty()
     status_container.empty()
     return all_data
 
+
 def convert_to_csv(data):
     df = pd.DataFrame(data["grants"])
     return df.to_csv(index=False).encode("utf-8")
 
+
 def convert_to_excel(data):
     df = pd.DataFrame(data["grants"])
     buffer = io.BytesIO()
@@ -85,170 +92,170 @@ def convert_to_excel(data):
         df.to_excel(writer, sheet_name="Grants", index=False)
     return buffer.getvalue()
 
+
 def create_knowledge_base(data):
     documents = []
     for grant in data["grants"]:
         doc_parts = [f"{key.replace('_', ' ').title()}: {value}" for key, value in grant.items()]
         documents.append("\n".join(doc_parts))
+
     text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
     texts = text_splitter.create_documents(documents)
+
     embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY)
     vectorstore = FAISS.from_documents(texts, embeddings)
+
     llm = ChatGoogleGenerativeAI(
         model="gemini-2.0-flash-thinking-exp", google_api_key=GOOGLE_API_KEY, temperature=0
     )
     memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
     return ConversationalRetrievalChain.from_llm(llm, vectorstore.as_retriever(), memory=memory)
 
+
 def get_shareable_link(file_data, file_name, file_type):
     b64 = base64.b64encode(file_data).decode()
     return f"data:{file_type};base64,{b64}"
 
-def display_dashboard(data):
-    df = pd.DataFrame(data["grants"])
-    if df.empty:
-        st.info("No data available for dashboard.")
-        return
-    st.subheader("Grants Dashboard")
-    # Filtering by sector if available
-    if "sector" in df.columns:
-        sectors = df["sector"].dropna().unique().tolist()
-        selected_sector = st.selectbox("Select Sector", options=["All"] + sectors)
-        if selected_sector != "All":
-            df = df[df["sector"] == selected_sector]
-    # Visualizations: Distribution of Grant Values
-    if "value" in df.columns:
-        df["value"] = pd.to_numeric(df["value"], errors="coerce")
-        fig_value = px.histogram(df, x="value", nbins=20, title="Distribution of Grant Values")
-        st.plotly_chart(fig_value)
-    # Visualization: Grants by Organisation
-    if "organisation" in df.columns:
-        fig_org = px.pie(df, names="organisation", title="Grants by Organisation")
-        st.plotly_chart(fig_org)
-    st.dataframe(df)
-
-def display_scrape_tab():
-    st.header("Scrape Grants")
-    url_input = st.text_area(
-        "Enter URLs (one per line)",
+
+def main():
+    st.set_page_config(page_title="Quantilytix Grant Finder", page_icon="💰", layout="wide")
+    st.title("💰 Quantilytix Grant Finder")
+
+    # --- Introduction and Motivation ---
+    st.markdown("""
+    <div style="text-align: justify;">
+        <p>
+            Welcome to <b>Quantilytix Grant Finder</b>, an AI-powered platform designed to streamline the grant discovery process, especially for academics and researchers in Zimbabwe and LMICs. 
+            In the evolving landscape of development, securing research funding is crucial, yet often challenging. Our platform leverages advanced AI to make grant searching more efficient and accessible.
+        </p>
+        <p>
+            <b>Inspired by the Quintuple Helix Model and the pressing need for research funding in LMICs</b>, this tool aims to bridge the gap between researchers with innovative ideas and funding opportunities worldwide. 
+            We understand the scarcity of research funds, particularly in regions like Zimbabwe, and are committed to providing a cost-effective, tailored solution.
+        </p>
+        <p>
+            <b>Future enhancements</b> will include personalized alerts and customized access models to cater to both individual academics and research offices, ensuring that relevant grant opportunities are never missed. 
+            Stay tuned for pricing assessments tailored for universities and research institutions in Zimbabwe, making advanced grant searching accessible to all.
+        </p>
+    </div>
+    """, unsafe_allow_html=True)
+
+    st.sidebar.image("logoqb.jpeg", use_container_width=True)
+    st.sidebar.header("Scrape & Configure")
+
+    # Initialize session state
+    if "scraped_data" not in st.session_state:
+        st.session_state.scraped_data = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+    if "chat_interface_active" not in st.session_state:
+        st.session_state.chat_interface_active = False
+
+    # URL Input in Sidebar
+    url_input = st.sidebar.text_area(
+        "Enter Grant URLs (one per line)",
         height=150,
-        help="Enter multiple URLs separated by new lines",
+        help="Input URLs from funding websites. Add each URL on a new line.",
+        placeholder="e.g.,\nhttps://www.example-grants.org/opportunities\nhttps://another-funding-source.com/grants-list"
     )
-    if st.button("Start Scraping"):
+
+    # Get Grants Button with Icon
+    if st.sidebar.button("🔍 Get Grant Opportunities"):
         if url_input:
             urls = [url.strip() for url in url_input.split("\n") if url.strip()]
             if urls:
-                with st.spinner("Scraping grants..."):
-                    result = process_multiple_urls(urls)
-                    st.session_state.scraped_data = result
-                    st.success(f"Scraped {len(result['grants'])} grants from {len(urls)} URL(s)!")
+                try:
+                    with st.spinner("Scraping in progress... Please wait patiently."):
+                        result = process_multiple_urls(urls)
+                        st.session_state.scraped_data = result
+                        st.success(f"✅ Successfully scraped {len(result['grants'])} grant opportunities from {len(urls)} URLs!")
+                except Exception as e:
+                    st.error(f"🚨 Scraping process encountered an error: {e}")
             else:
-                st.warning("Please enter valid URLs.")
-        else:
-            st.warning("Please enter at least one URL.")
-
-def display_download_tab():
-    st.header("Download & Explore Data")
-    if st.session_state.get("scraped_data"):
-        result = st.session_state.scraped_data
-        df = pd.DataFrame(result["grants"])
-        st.subheader(f"Data Preview ({len(df)} grants)")
-        st.dataframe(df)
-        selected_format = st.selectbox("Select Download Format", ("CSV", "Excel"))
-        if selected_format == "CSV":
-            file_data = convert_to_csv(result)
-            file_name = "grants.csv"
-            file_type = "text/csv"
+                st.warning("⚠️ Please enter valid URLs.")
         else:
-            file_data = convert_to_excel(result)
-            file_name = "grants.xlsx"
-            file_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
-        b64 = base64.b64encode(file_data).decode()
-        download_link = f"<a href='data:{file_type};base64,{b64}' download='{file_name}'>Download {selected_format}</a>"
-        st.markdown(download_link, unsafe_allow_html=True)
-        shareable_link = get_shareable_link(file_data, file_name, file_type)
-        st.markdown("---")
-        st.markdown("**Share Options:**")
-        whatsapp_url = f"https://api.whatsapp.com/send?text={urllib.parse.quote(f'Check out this file: {shareable_link}')}"
-        st.markdown(f"📱 [Share via WhatsApp]({whatsapp_url})")
-        email_subject = urllib.parse.quote("Check out this grants file")
-        email_body = urllib.parse.quote(f"Download the file here: {shareable_link}")
-        email_url = f"mailto:?subject={email_subject}&body={email_body}"
-        st.markdown(f"📧 [Share via Email]({email_url})")
-    else:
-        st.info("No scraped data available. Please scrape grants first.")
-
-def display_chat_tab():
-    st.header("Knowledge Base Chat")
-    st.info("Ask questions about the grants data.")
-    if st.session_state.get("scraped_data"):
-        if st.button("Load Data as Knowledge Base"):
-            st.session_state.qa_chain = create_knowledge_base(st.session_state.scraped_data)
-            st.session_state.chat_interface_active = True
-            st.session_state.chat_history = []
-            st.success("Knowledge base loaded!")
+            st.warning("⚠️ Please enter at least one URL to begin scraping.")
+
+    # --- Main Panel for Data Display and Chat ---
+    st.markdown("---")
+
+    if st.session_state.scraped_data and st.session_state.scraped_data['grants']:
+        st.header("📊 Scraped Grant Data")
+
+        # Data Preview and Download Options in Main Panel
+        with st.expander(f"📊 Preview Grant Data ({len(st.session_state.scraped_data['grants']} grants)"):
+            st.dataframe(st.session_state.scraped_data["grants"])
+
+        col1, col2, col3 = st.columns([1, 1, 2]) # Adjust column widths for better layout
+
+        with col1:
+            selected_format = st.selectbox("Download As:", ("CSV", "Excel"), key="download_format_selector")
+
+        with col2:
+            if selected_format == "CSV":
+                file_data = convert_to_csv(st.session_state.scraped_data)
+                file_name = "grants_data.csv"
+                file_type = "text/csv"
+            else:
+                file_data = convert_to_excel(st.session_state.scraped_data)
+                file_name = "grants_data.xlsx"
+                file_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
+
+            download_link_html = f"<a href='data:{file_type};base64,{base64.b64encode(file_data).decode()}' download='{file_name}'><button style='background-color:#4CAF50;color:white;padding:10px 15px;border:none;border-radius:4px;'>⬇️ Download {selected_format}</button></a>"
+            st.markdown(download_link_html, unsafe_allow_html=True)
+
+        with col3:
+            shareable_link = get_shareable_link(file_data, file_name, file_type)
+            whatsapp_url = f"https://api.whatsapp.com/send?text={urllib.parse.quote(f'Check out these grant opportunities: {shareable_link}')}"
+            email_subject = urllib.parse.quote("Grant Opportunities File")
+            email_body = urllib.parse.quote(f"Download the grant opportunities file here: {shareable_link}")
+            email_url = f"mailto:?subject={email_subject}&body={email_body}"
+
+            st.markdown("<div style='margin-top:10px;'>Share via:</div>", unsafe_allow_html=True) # Add some margin for better spacing
+            st.markdown(f"📱 [WhatsApp]({whatsapp_url}) | 📧 [Email]({email_url})", unsafe_allow_html=True)
+
+
+        # Knowledge Base and Chat Interface
+        if st.button("🧠 Load as Knowledge Base & Chat"):
+            with st.spinner("Loading data into knowledge base..."):
+                st.session_state.qa_chain = create_knowledge_base(st.session_state.scraped_data)
+                st.session_state.chat_interface_active = True
+                st.session_state.chat_history = [] # Clear chat history on reload
+            st.success("Knowledge base loaded! You can now chat with the Grants Bot.")
+
         if st.session_state.get("chat_interface_active"):
-            query = st.text_input("Enter your query:")
+            st.markdown("---")
+            st.header("💬 Chat with Grants Bot")
+            st.markdown("Ask questions about the scraped grants to get quick insights!")
+
+            query = st.text_input("Your question:", key="chat_input")
             if query:
-                response = st.session_state.qa_chain({"question": query})
-                st.session_state.chat_history.append({"query": query, "response": response["answer"]})
-            for chat in st.session_state.get("chat_history", []):
-                st.markdown(f"**You:** {chat['query']}")
-                st.markdown(f"**Grants Bot:** {chat['response']}")
-        else:
-            st.info("Load the knowledge base to start chatting.")
+                if st.session_state.qa_chain:
+                    with st.spinner("Generating response..."):
+                        response = st.session_state.qa_chain({"question": query})
+                        st.session_state.chat_history.append({"query": query, "response": response["answer"]})
+                else:
+                    st.error("Knowledge base not initialized. Please load data as knowledge base.")
+
+            if st.session_state.chat_history:
+                st.subheader("Chat History")
+                for chat in st.session_state.chat_history:
+                    st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 5px; background-color: #f0f2f6;'><strong>You:</strong> {chat['query']}</div>", unsafe_allow_html=True)
+                    st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 10px; background-color: #e0e2e6;'><strong>Grants Bot:</strong> {chat['response']}</div>", unsafe_allow_html=True)
+
+
     else:
-        st.info("No scraped data available. Please scrape grants first.")
-
-def display_alerts_tab():
-    st.header("Automated Alerts Setup")
-    st.write("Configure your personalized alerts for new grant opportunities.")
-    keyword = st.text_input("Keyword for Alerts", help="E.g., 'AI', 'sustainable research'")
-    sector = st.text_input("Sector", help="E.g., 'health', 'technology'")
-    email = st.text_input("Your Email", help="Enter your email to receive alerts")
-    if st.button("Save Alert Preferences"):
-        # Placeholder for saving alert preferences—integrate with an email service in a full implementation.
-        st.success("Alert preferences saved! You will be notified when matching grants are found.")
-    st.info("Note: Automated alert functionality is under development and will be integrated soon.")
+        st.info("⬅️  Enter URLs in the sidebar and click 'Get Grant Opportunities' to start scraping.")
+
+    st.sidebar.markdown("---")
+    st.sidebar.markdown(
+        """
+        <div style='text-align: center; font-size: 0.8em; color: grey;'>
+            Powered by <a href="https://quantilytix.com" style='color: grey;'>Quantilytix</a> | &copy; 2025
+        </div>
+        """,
+        unsafe_allow_html=True,
+    )
 
-def main():
-    st.set_page_config(page_title="Quantilytix Grants Platform", layout="wide", initial_sidebar_state="expanded")
-    # Custom CSS styling for a modern look
-    st.markdown("""
-    <style>
-    .main {
-        background-color: #f5f5f5;
-    }
-    .sidebar .sidebar-content {
-        background-image: linear-gradient(#2e7bcf, #2e7bcf);
-        color: white;
-    }
-    </style>
-    """, unsafe_allow_html=True)
-    
-    st.sidebar.title("Quantilytix Grants Platform")
-    st.sidebar.image("logoqb.jpeg", use_column_width=True)
-    app_mode = st.sidebar.radio("Navigation", ["Scrape Grants", "Download & Explore", "Dashboard", "Knowledge Base Chat", "Automated Alerts"])
-    
-    if app_mode == "Scrape Grants":
-        display_scrape_tab()
-    elif app_mode == "Download & Explore":
-        display_download_tab()
-    elif app_mode == "Dashboard":
-        if st.session_state.get("scraped_data"):
-            display_dashboard(st.session_state.scraped_data)
-        else:
-            st.info("No data available. Please scrape grants first.")
-    elif app_mode == "Knowledge Base Chat":
-        display_chat_tab()
-    elif app_mode == "Automated Alerts":
-        display_alerts_tab()
 
 if __name__ == "__main__":
-    if "scraped_data" not in st.session_state:
-        st.session_state.scraped_data = None
-    if "chat_history" not in st.session_state:
-        st.session_state.chat_history = []
-    if "chat_interface_active" not in st.session_state:
-        st.session_state.chat_interface_active = False
     main()