Update app.py

app.py

@@ -2,7 +2,7 @@ import streamlit as st
 import pandas as pd
 import base64
 import json
-from scrapegraphai.graphs import SmartScraperGraph, SearchGraph
+from scrapegraphai.graphs import SearchGraph
 import nest_asyncio
 import os
 import subprocess
@@ -25,73 +25,62 @@ graph_config = {
         "api_key": GOOGLE_API_KEY,
         "model": "google_genai/gemini-2.0-flash-thinking-exp",
     },
+    "max_results": 2,
+    "verbose": True,
 }
 
-
-def get_data(url):
-    smart_scraper_graph = SmartScraperGraph(
-        prompt=(
-            "List me all grants or funds with: "
-            "- Grant name/title\n"
-            "- Short summary (50-100 characters)\n"
-            "- Funding organization\n"
-            "- Grant value (numeric only)\n"
-            "- Application deadline\n"
-            "- Eligible countries\n"
-            "- Sector/field\n"
-            "- Eligibility criteria\n"
-            "Return in JSON format."
-        ),
-        source=url,
+def get_data(search_term):
+    # Combine the user input with 'grants'
+    full_prompt = f"search for {search_term} grants\n\nList me all grants or funds with:\n- Grant name/title\n- Short summary (50-100 characters)\n- Funding organization\n- Grant value (numeric only)\n- Application deadline\n- Eligible countries\n- Sector/field\n- Eligibility criteria\nReturn in JSON format."
+    search_graph = SearchGraph(
+        prompt=full_prompt,
         config=graph_config,
     )
-    return smart_scraper_graph.run()
-
+    return search_graph.run()
 
-def process_multiple_urls(urls):
+def process_multiple_search_terms(search_terms):
     """
-    Process multiple URLs with enhanced progress tracking and user feedback.
+    Process multiple search terms with enhanced progress tracking and user feedback.
     """
     all_data = {"grants": []}
     progress_bar = st.progress(0)
     status_container = st.empty()
-    total_urls = len(urls)
+    total_terms = len(search_terms)
 
-    for index, url in enumerate(urls):
+    for index, term in enumerate(search_terms):
         try:
-            url = url.strip()
-            if not url:
+            term = term.strip()
+            if not term:
                 continue
 
-            progress = (index + 1) / total_urls
+            progress = (index + 1) / total_terms
             progress_bar.progress(progress)
             status_container.markdown(
                 f"""
-    **Processing Grant Opportunities** 🚀
-    Scanning URL {index+1} of {total_urls}:  `{url}`
-    <br>
-    <p style='font-size: 0.9em; color: #6699CC;'>Completed: {index}/{total_urls} | Remaining: {total_urls - index - 1}</p>
-    """,
+**Processing Grant Opportunities** 🚀
+Searching term {index+1} of {total_terms}:  `{term}`
+<br>
+<p style='font-size: 0.9em; color: #6699CC;'>Completed: {index}/{total_terms} | Remaining: {total_terms - index - 1}</p>
+""",
                 unsafe_allow_html=True,
             )
 
-            result = get_data(url)
+            result = get_data(term)
+            # Expecting a JSON with a "grants" key
             if result and "grants" in result:
                 all_data["grants"].extend(result["grants"])
         except Exception as e:
-            st.error(f"⚠️ Error processing URL: {url} - {str(e)}")
+            st.error(f"⚠️ Error processing search term: {term} - {str(e)}")
             continue
 
     progress_bar.empty()
     status_container.empty()
     return all_data
 
-
 def convert_to_csv(data):
     df = pd.DataFrame(data["grants"])
     return df.to_csv(index=False).encode("utf-8")
 
-
 def convert_to_excel(data):
     df = pd.DataFrame(data["grants"])
     buffer = io.BytesIO()
@@ -99,26 +88,24 @@ def convert_to_excel(data):
         df.to_excel(writer, sheet_name="Grants", index=False)
     return buffer.getvalue()
 
-
 def create_knowledge_base(data):
     # Store JSON representation of data in session state
     st.session_state.knowledge_base_json = json.dumps(data, indent=2)
 
-
 def chat_with_knowledge_base(query):
     if "knowledge_base_json" not in st.session_state:
         return "Knowledge base not initialized. Please load grant data first."
 
     context = st.session_state.knowledge_base_json
     prompt = f"""
-    You are an AI assistant that helps users analyze grant opportunities.
-    Here is the extracted grant data in JSON format:
-    
-    {context}
+You are an AI assistant that helps users analyze grant opportunities.
+Here is the extracted grant data in JSON format:
 
-    User's question: {query}
-    Answer the question based on the provided grant data.
-    """
+{context}
+
+User's question: {query}
+Answer the question based on the provided grant data.
+"""
 
     llm = ChatGoogleGenerativeAI(
         model="gemini-2.0-flash-thinking-exp", google_api_key=GOOGLE_API_KEY, temperature=0
@@ -127,30 +114,25 @@ def chat_with_knowledge_base(query):
     response = llm.invoke(prompt)
     return response
 
-
 def get_shareable_link(file_data, file_name, file_type):
     b64 = base64.b64encode(file_data).decode()
     return f"data:{file_type};base64,{b64}"
 
-
 def main():
     st.set_page_config(page_title="Quantilytix Grant Finder", page_icon="💰", layout="wide")
     st.title("💰 Quantilytix Grant Finder")
 
-    # --- Introduction and Motivation ---
     st.markdown("""
-    <div style="text-align: justify;">
-        <p>
-            Welcome to <b>Quantilytix Grant Finder</b>, an AI-powered platform designed to streamline the grant discovery process, especially for academics and researchers across the globe.
-        </p>
-            
-    </div>
-    """, unsafe_allow_html=True)
+<div style="text-align: justify;">
+    <p>
+        Welcome to <b>Quantilytix Grant Finder</b>, an AI-powered platform designed to streamline the grant discovery process, especially for academics and researchers across the globe.
+    </p>
+</div>
+""", unsafe_allow_html=True)
 
     st.sidebar.image("logoqb.jpeg", use_container_width=True)
     st.sidebar.header("Scrape & Configure")
 
-    # Initialize session state
     if "scraped_data" not in st.session_state:
         st.session_state.scraped_data = None
     if "chat_history" not in st.session_state:
@@ -158,42 +140,39 @@ def main():
     if "chat_interface_active" not in st.session_state:
         st.session_state.chat_interface_active = False
 
-    # URL Input in Sidebar
-    url_input = st.sidebar.text_area(
-        "Enter Grant URLs (one per line)",
+    # Search Term Input in Sidebar
+    search_input = st.sidebar.text_area(
+        "Enter Search Terms (one per line)",
         height=150,
-        help="Input URLs from funding websites. Add each URL on a new line.",
-        placeholder="e.g.,\nhttps://www.example-grants.org/opportunities\nhttps://another-funding-source.com/grants-list"
+        help="Input search terms to discover grant opportunities. Each term will be combined with 'grants' for the search.",
+        placeholder="e.g.,\neducation\nresearch\ntechnology"
     )
 
-    # Get Grants Button with Icon
     if st.sidebar.button("🔍 Get Grant Opportunities"):
-        if url_input:
-            urls = [url.strip() for url in url_input.split("\n") if url.strip()]
-            if urls:
+        if search_input:
+            search_terms = [term.strip() for term in search_input.split("\n") if term.strip()]
+            if search_terms:
                 try:
-                    with st.spinner("Scraping in progress... Please wait patiently."):
-                        result = process_multiple_urls(urls)
+                    with st.spinner("Searching in progress... Please wait patiently."):
+                        result = process_multiple_search_terms(search_terms)
                         st.session_state.scraped_data = result
-                        st.success(f"✅ Successfully scraped {len(result['grants'])} grant opportunities from {len(urls)} URLs!")
+                        st.success(f"✅ Successfully found {len(result['grants'])} grant opportunities from {len(search_terms)} search terms!")
                 except Exception as e:
-                    st.error(f"🚨 Scraping process encountered an error: {e}")
+                    st.error(f"🚨 Searching process encountered an error: {e}")
             else:
-                st.warning("⚠️ Please enter valid URLs.")
+                st.warning("⚠️ Please enter valid search terms.")
         else:
-            st.warning("⚠️ Please enter at least one URL to begin scraping.")
+            st.warning("⚠️ Please enter at least one search term to begin.")
 
-    # --- Main Panel for Data Display and Chat ---
     st.markdown("---")
 
     if st.session_state.scraped_data and st.session_state.scraped_data['grants']:
-        st.header("📊 Scraped Grant Data")
+        st.header("📊 Found Grant Data")
 
-        # Data Preview and Download Options in Main Panel
-        with st.expander(f"📊 Preview Grant Data {len(st.session_state.scraped_data['grants'])} grants"):
+        with st.expander(f"📊 Preview Grant Data ({len(st.session_state.scraped_data['grants'])} grants)"):
             st.dataframe(st.session_state.scraped_data["grants"])
 
-        col1, col2, col3 = st.columns([1, 1, 2]) # Adjust column widths for better layout
+        col1, col2, col3 = st.columns([1, 1, 2])
 
         with col1:
             selected_format = st.selectbox("Download As:", ("CSV", "Excel"), key="download_format_selector")
@@ -218,28 +197,28 @@ def main():
             email_body = urllib.parse.quote(f"Download the grant opportunities file here: {shareable_link}")
             email_url = f"mailto:?subject={email_subject}&body={email_body}"
 
-            st.markdown("<div style='margin-top:10px;'>Share via:</div>", unsafe_allow_html=True) # Add some margin for better spacing
+            st.markdown("<div style='margin-top:10px;'>Share via:</div>", unsafe_allow_html=True)
             st.markdown(f"📱 [WhatsApp]({whatsapp_url}) | 📧 [Email]({email_url})", unsafe_allow_html=True)
 
-
-        # Knowledge Base and Chat Interface
         if st.button("🧠 Load as Knowledge Base & Chat"):
             with st.spinner("Loading data into knowledge base..."):
-                st.session_state.qa_chain = create_knowledge_base(st.session_state.scraped_data)
+                create_knowledge_base(st.session_state.scraped_data)
                 st.session_state.chat_interface_active = True
-                st.session_state.chat_history = [] # Clear chat history on reload
+                st.session_state.chat_history = []
             st.success("Knowledge base loaded! You can now chat with the Grants Bot.")
 
         if st.session_state.get("chat_interface_active"):
             st.markdown("---")
             st.header("💬 Chat with Grants Bot")
-            st.markdown("Ask questions about the scraped grants to get quick insights!")
+            st.markdown("Ask questions about the found grants to get quick insights!")
 
             query = st.text_input("Your question:", key="chat_input")
             if query:
                 with st.spinner("Generating response..."):
                     response = chat_with_knowledge_base(query)
-                    st.session_state.chat_history.append({"query": query, "response": response["answer"]})
+                    # Assume response returns a dictionary with key 'answer'
+                    answer = response["answer"] if isinstance(response, dict) and "answer" in response else response
+                    st.session_state.chat_history.append({"query": query, "response": answer})
 
             if st.session_state.chat_history:
                 st.subheader("Chat History")
@@ -247,17 +226,16 @@ def main():
                     st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 5px; background-color: #f0f2f6;'><strong>You:</strong> {chat['query']}</div>", unsafe_allow_html=True)
                     st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 10px; background-color: #e0e2e6;'><strong>Grants Bot:</strong> {chat['response']}</div>", unsafe_allow_html=True)
 
-
     else:
-        st.info("⬅️  Enter URLs in the sidebar and click 'Get Grant Opportunities' to start scraping.")
+        st.info("⬅️  Enter search terms in the sidebar and click 'Get Grant Opportunities' to start searching.")
 
     st.sidebar.markdown("---")
     st.sidebar.markdown(
         """
-        <div style='text-align: center; font-size: 0.8em; color: grey;'>
-            Powered by <a href="https://quantilytix.com" style='color: grey;'>Quantilytix</a> | &copy; 2025
-        </div>
-        """,
+<div style='text-align: center; font-size: 0.8em; color: grey;'>
+    Powered by <a href="https://quantilytix.com" style='color: grey;'>Quantilytix</a> | &copy; 2025
+</div>
+""",
         unsafe_allow_html=True,
     )