Update app.py

app.py

@@ -8,19 +8,22 @@ import os
 import subprocess
 import io
 import time
+import urllib.parse
 from langchain_google_genai import ChatGoogleGenerativeAI, GoogleGenerativeAIEmbeddings
 from langchain.vectorstores import FAISS
 from langchain.text_splitter import CharacterTextSplitter
 from langchain.chains import ConversationalRetrievalChain
 from langchain.memory import ConversationBufferMemory
-import urllib.parse
+
+# Import Supadata and initialize the client
+from supadata import Supadata, SupadataError
+SUPADATA_API_KEY = os.getenv("SUPADATA")
+supadata = Supadata(api_key=SUPADATA_API_KEY)
 
 # Ensure Playwright installs required browsers and dependencies
 subprocess.run(["playwright", "install"])
 nest_asyncio.apply()
 
-
-
 GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"]
 
 graph_config = {
@@ -31,7 +34,6 @@ graph_config = {
     "max_results": 8,
     "verbose": True,
     "headless": True
-    
 }
 
 def get_data(search_term):
@@ -60,7 +62,7 @@ def get_data(search_term):
         )
         result = search_graph.run()
         if not result or not result.get("grants"):
-            st.error(f"No results returned for {search_term}. Please try again with a different search_term.")
+            st.error(f"No results returned for {search_term}. Please try again with a different search term.")
             return {}
         return result
     except Exception as e:
@@ -82,8 +84,47 @@ def get_data(search_term):
                 st.error(f"Retry failed: {e2}. Please try again later.")
                 return {}
         else:
-            st.error(f"An error occurred for search term : {search_term}, error: {e}. Please try again.")
+            st.error(f"An error occurred for search term: {search_term}, error: {e}. Please try again.")
+            return {}
+
+def get_data_from_url(url):
+    """
+    Scrape the provided URL using Supadata and pass the page content directly to the Gemini model
+    (using ChatGoogleGenerativeAI) to extract grant data in a JSON structure.
+    """
+    try:
+        web_content = supadata.web.scrape(url)
+        page_content = web_content.content
+        full_prompt = (
+            "Extract the following grant data from the provided web content. "
+            "List me all grants or funds with:\n"
+            "- Grant name/title\n"
+            "- Short summary\n"
+            "- Funding organization\n"
+            "- Grant value (numeric only)\n"
+            "- Application deadline\n"
+            "- Eligible countries\n"
+            "- Sector/field\n"
+            "- Eligibility criteria\n"
+            "Return in JSON format.\n\n"
+            f"Web content: {page_content}"
+        )
+        llm = ChatGoogleGenerativeAI(
+            model="gemini-2.0-flash-thinking-exp", google_api_key=GOOGLE_API_KEY, temperature=0
+        )
+        response = llm.invoke(full_prompt)
+        try:
+            result = json.loads(response.content)
+        except Exception as parse_error:
+            st.error("Error parsing JSON from Gemini model response.")
+            return {}
+        if not result or not result.get("grants"):
+            st.error("No grant opportunities found in the scraped URL.")
             return {}
+        return result
+    except Exception as e:
+        st.error(f"An error occurred while scraping URL {url}: {e}")
+        return {}
 
 def process_multiple_search_terms(search_terms):
     """
@@ -182,27 +223,52 @@ def main():
     if "chat_interface_active" not in st.session_state:
         st.session_state.chat_interface_active = False
 
-    # Sidebar: Search Term Input
-    search_input = st.sidebar.text_area(
-        "Enter Search Terms (one per line). maximum 2",
-        height=150,
-        help="Input search terms to discover grant opportunities. Terms can be specific or generic.",
-        placeholder="e.g.,\nRenewable energy \nclimate change research\nAgriculture in Africa"
+    # Sidebar: Input Type Selection
+    input_type = st.sidebar.radio(
+        "Select Input Type:",
+        ("Search Query", "URL"),
+        key="input_type_selector"
     )
 
-    if st.sidebar.button("🔍 Get Grant Opportunities"):
-        if search_input:
-            search_terms = [term.strip() for term in search_input.split("\n") if term.strip()]
-            if search_terms:
-                with st.spinner("Searching in progress... Please wait patiently."):
-                    result = process_multiple_search_terms(search_terms)
+    # Sidebar: Input field based on selection
+    if input_type == "Search Query":
+        search_input = st.sidebar.text_area(
+            "Enter Search Terms (one per line). Maximum 2",
+            height=150,
+            help="Input search terms to discover grant opportunities. Terms can be specific or generic.",
+            placeholder="e.g.,\nRenewable energy \nclimate change research\nAgriculture in Africa"
+        )
+    else:
+        url_input = st.sidebar.text_input(
+            "Enter URL to scrape for grant opportunities",
+            placeholder="https://example.com/grants"
+        )
+
+    # Execute based on input type selection
+    if input_type == "Search Query":
+        if st.sidebar.button("🔍 Get Grant Opportunities"):
+            if search_input:
+                search_terms = [term.strip() for term in search_input.split("\n") if term.strip()]
+                if search_terms:
+                    with st.spinner("Searching in progress... Please wait patiently."):
+                        result = process_multiple_search_terms(search_terms)
+                        st.session_state.scraped_data = result
+                        if result.get("grants"):
+                            st.sidebar.success(f"✅ Found {len(result['grants'])} grant opportunities from {len(search_terms)} search terms!")
+                else:
+                    st.sidebar.warning("⚠️ Please enter valid search terms.")
+            else:
+                st.sidebar.warning("⚠️ Please enter at least one search term to begin.")
+    else:  # URL input
+        if st.sidebar.button("🔍 Scrape URL for Grant Opportunities"):
+            if url_input:
+                with st.spinner("Scraping URL... Please wait patiently."):
+                    result = get_data_from_url(url_input)
                     st.session_state.scraped_data = result
                     if result.get("grants"):
-                        st.sidebar.success(f"✅ Found {len(result['grants'])} grant opportunities from {len(search_terms)} search terms!")
+                        st.sidebar.success(f"✅ Found {len(result['grants'])} grant opportunities from the URL!")
             else:
-                st.sidebar.warning("⚠️ Please enter valid search terms.")
-        else:
-            st.sidebar.warning("⚠️ Please enter at least one search term to begin.")
+                st.sidebar.warning("⚠️ Please enter a valid URL to scrape.")
 
     # Sidebar: Download & Share Controls
     if st.session_state.scraped_data and st.session_state.scraped_data.get('grants'):
@@ -254,18 +320,15 @@ def main():
         
         if st.session_state.chat_history:
             st.subheader("Chat History")
-            # Reverse the chat history to show the latest messages first
             for chat in reversed(st.session_state.chat_history):
-                # User message: dark grey background with white text
                 st.markdown(
                     f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 5px; background-color:#444444; color: white;'><strong>You:</strong> {chat['query']}</div>",
                     unsafe_allow_html=True)
-                # Bot message: blue background with white text
                 st.markdown(
                     f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 10px; background-color:#007BFF; color: white;'><strong>Grants Bot:</strong> {chat['response']}</div>",
                     unsafe_allow_html=True)
     else:
-        st.info("⬅️  Enter search terms in the sidebar and click 'Get Grant Opportunities' to start searching.")
+        st.info("⬅️  Enter search terms or a URL in the sidebar and click the appropriate button to start searching.")
 
     st.sidebar.markdown("---")
     st.sidebar.markdown(
@@ -278,4 +341,4 @@ def main():
     )
 
 if __name__ == "__main__":
-    main()
+    main()