Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import pandas as pd | |
| import base64 | |
| import json | |
| from scrapegraphai.graphs import SmartScraperGraph | |
| import nest_asyncio | |
| import os | |
| import subprocess | |
| import io | |
| from langchain_google_genai import ChatGoogleGenerativeAI, GoogleGenerativeAIEmbeddings | |
| from langchain.vectorstores import FAISS | |
| from langchain.text_splitter import CharacterTextSplitter | |
| from langchain.chains import ConversationalRetrievalChain | |
| from langchain.memory import ConversationBufferMemory | |
| import urllib.parse | |
| # Ensure Playwright installs required browsers and dependencies | |
| subprocess.run(["playwright", "install"]) | |
| nest_asyncio.apply() | |
| GOOGLE_API_KEY = os.environ["GOOGLE_API_KEY"] | |
| graph_config = { | |
| "llm": { | |
| "api_key": GOOGLE_API_KEY, | |
| "model": "google_genai/gemini-2.0-flash-thinking-exp", | |
| }, | |
| } | |
| def get_data(url): | |
| smart_scraper_graph = SmartScraperGraph( | |
| prompt=( | |
| "List me all grants or funds, short summary of grant description, " | |
| "the organisations funding them, the value of the grant as an integer, " | |
| "the due date, eligible countries, sector and eligibility criteria for applicants." | |
| ), | |
| source=url, | |
| config=graph_config, | |
| ) | |
| return smart_scraper_graph.run() | |
| def process_multiple_urls(urls): | |
| """ | |
| Process multiple URLs with enhanced progress tracking and user feedback. | |
| """ | |
| all_data = {"grants": []} | |
| progress_bar = st.progress(0) | |
| status_container = st.empty() | |
| total_urls = len(urls) | |
| for index, url in enumerate(urls): | |
| try: | |
| url = url.strip() | |
| if not url: | |
| continue | |
| progress = (index + 1) / total_urls | |
| progress_bar.progress(progress) | |
| status_container.markdown( | |
| f""" | |
| **Processing Grant Opportunities** 🚀 | |
| Scanning URL {index+1} of {total_urls}: `{url}` | |
| <br> | |
| <p style='font-size: 0.9em; color: #6699CC;'>Completed: {index}/{total_urls} | Remaining: {total_urls - index - 1}</p> | |
| """, | |
| unsafe_allow_html=True, | |
| ) | |
| result = get_data(url) | |
| if result and "grants" in result: | |
| all_data["grants"].extend(result["grants"]) | |
| except Exception as e: | |
| st.error(f"⚠️ Error processing URL: {url} - {str(e)}") | |
| continue | |
| progress_bar.empty() | |
| status_container.empty() | |
| return all_data | |
| def convert_to_csv(data): | |
| df = pd.DataFrame(data["grants"]) | |
| return df.to_csv(index=False).encode("utf-8") | |
| def convert_to_excel(data): | |
| df = pd.DataFrame(data["grants"]) | |
| buffer = io.BytesIO() | |
| with pd.ExcelWriter(buffer, engine="xlsxwriter") as writer: | |
| df.to_excel(writer, sheet_name="Grants", index=False) | |
| return buffer.getvalue() | |
| def create_knowledge_base(data): | |
| documents = [] | |
| for grant in data["grants"]: | |
| doc_parts = [f"{key.replace('_', ' ').title()}: {value}" for key, value in grant.items()] | |
| documents.append("\n".join(doc_parts)) | |
| text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200) | |
| texts = text_splitter.create_documents(documents) | |
| embeddings = GoogleGenerativeAIEmbeddings(model="models/embedding-001", google_api_key=GOOGLE_API_KEY) | |
| vectorstore = FAISS.from_documents(texts, embeddings) | |
| llm = ChatGoogleGenerativeAI( | |
| model="gemini-2.0-flash-thinking-exp", google_api_key=GOOGLE_API_KEY, temperature=0 | |
| ) | |
| memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) | |
| return ConversationalRetrievalChain.from_llm(llm, vectorstore.as_retriever(), memory=memory) | |
| def get_shareable_link(file_data, file_name, file_type): | |
| b64 = base64.b64encode(file_data).decode() | |
| return f"data:{file_type};base64,{b64}" | |
| def main(): | |
| st.set_page_config(page_title="Quantilytix Grant Finder", page_icon="💰", layout="wide") | |
| st.title("💰 Quantilytix Grant Finder") | |
| # --- Introduction and Motivation --- | |
| st.markdown(""" | |
| <div style="text-align: justify;"> | |
| <p> | |
| Welcome to <b>Quantilytix Grant Finder</b>, an AI-powered platform designed to streamline the grant discovery process, especially for academics and researchers in Zimbabwe. | |
| </p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| st.sidebar.image("logoqb.jpeg", use_container_width=True) | |
| st.sidebar.header("Scrape & Configure") | |
| # Initialize session state | |
| if "scraped_data" not in st.session_state: | |
| st.session_state.scraped_data = None | |
| if "chat_history" not in st.session_state: | |
| st.session_state.chat_history = [] | |
| if "chat_interface_active" not in st.session_state: | |
| st.session_state.chat_interface_active = False | |
| # URL Input in Sidebar | |
| url_input = st.sidebar.text_area( | |
| "Enter Grant URLs (one per line)", | |
| height=150, | |
| help="Input URLs from funding websites. Add each URL on a new line.", | |
| placeholder="e.g.,\nhttps://www.example-grants.org/opportunities\nhttps://another-funding-source.com/grants-list" | |
| ) | |
| # Get Grants Button with Icon | |
| if st.sidebar.button("🔍 Get Grant Opportunities"): | |
| if url_input: | |
| urls = [url.strip() for url in url_input.split("\n") if url.strip()] | |
| if urls: | |
| try: | |
| with st.spinner("Scraping in progress... Please wait patiently."): | |
| result = process_multiple_urls(urls) | |
| st.session_state.scraped_data = result | |
| st.success(f"✅ Successfully scraped {len(result['grants'])} grant opportunities from {len(urls)} URLs!") | |
| except Exception as e: | |
| st.error(f"🚨 Scraping process encountered an error: {e}") | |
| else: | |
| st.warning("⚠️ Please enter valid URLs.") | |
| else: | |
| st.warning("⚠️ Please enter at least one URL to begin scraping.") | |
| # --- Main Panel for Data Display and Chat --- | |
| st.markdown("---") | |
| if st.session_state.scraped_data and st.session_state.scraped_data['grants']: | |
| st.header("📊 Scraped Grant Data") | |
| # Data Preview and Download Options in Main Panel | |
| with st.expander(f"📊 Preview Grant Data {len(st.session_state.scraped_data['grants'])} grants"): | |
| st.dataframe(st.session_state.scraped_data["grants"]) | |
| col1, col2, col3 = st.columns([1, 1, 2]) # Adjust column widths for better layout | |
| with col1: | |
| selected_format = st.selectbox("Download As:", ("CSV", "Excel"), key="download_format_selector") | |
| with col2: | |
| if selected_format == "CSV": | |
| file_data = convert_to_csv(st.session_state.scraped_data) | |
| file_name = "grants_data.csv" | |
| file_type = "text/csv" | |
| else: | |
| file_data = convert_to_excel(st.session_state.scraped_data) | |
| file_name = "grants_data.xlsx" | |
| file_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" | |
| download_link_html = f"<a href='data:{file_type};base64,{base64.b64encode(file_data).decode()}' download='{file_name}'><button style='background-color:#4CAF50;color:white;padding:10px 15px;border:none;border-radius:4px;'>⬇️ Download {selected_format}</button></a>" | |
| st.markdown(download_link_html, unsafe_allow_html=True) | |
| with col3: | |
| shareable_link = get_shareable_link(file_data, file_name, file_type) | |
| whatsapp_url = f"https://api.whatsapp.com/send?text={urllib.parse.quote(f'Check out these grant opportunities: {shareable_link}')}" | |
| email_subject = urllib.parse.quote("Grant Opportunities File") | |
| email_body = urllib.parse.quote(f"Download the grant opportunities file here: {shareable_link}") | |
| email_url = f"mailto:?subject={email_subject}&body={email_body}" | |
| st.markdown("<div style='margin-top:10px;'>Share via:</div>", unsafe_allow_html=True) # Add some margin for better spacing | |
| st.markdown(f"📱 [WhatsApp]({whatsapp_url}) | 📧 [Email]({email_url})", unsafe_allow_html=True) | |
| # Knowledge Base and Chat Interface | |
| if st.button("🧠 Load as Knowledge Base & Chat"): | |
| with st.spinner("Loading data into knowledge base..."): | |
| st.session_state.qa_chain = create_knowledge_base(st.session_state.scraped_data) | |
| st.session_state.chat_interface_active = True | |
| st.session_state.chat_history = [] # Clear chat history on reload | |
| st.success("Knowledge base loaded! You can now chat with the Grants Bot.") | |
| if st.session_state.get("chat_interface_active"): | |
| st.markdown("---") | |
| st.header("💬 Chat with Grants Bot") | |
| st.markdown("Ask questions about the scraped grants to get quick insights!") | |
| query = st.text_input("Your question:", key="chat_input") | |
| if query: | |
| if st.session_state.qa_chain: | |
| with st.spinner("Generating response..."): | |
| response = st.session_state.qa_chain({"question": query}) | |
| st.session_state.chat_history.append({"query": query, "response": response["answer"]}) | |
| else: | |
| st.error("Knowledge base not initialized. Please load data as knowledge base.") | |
| if st.session_state.chat_history: | |
| st.subheader("Chat History") | |
| for chat in st.session_state.chat_history: | |
| st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 5px; background-color: #f0f2f6;'><strong>You:</strong> {chat['query']}</div>", unsafe_allow_html=True) | |
| st.markdown(f"<div style='padding: 10px; border-radius: 5px; margin-bottom: 10px; background-color: #e0e2e6;'><strong>Grants Bot:</strong> {chat['response']}</div>", unsafe_allow_html=True) | |
| else: | |
| st.info("⬅️ Enter URLs in the sidebar and click 'Get Grant Opportunities' to start scraping.") | |
| st.sidebar.markdown("---") | |
| st.sidebar.markdown( | |
| """ | |
| <div style='text-align: center; font-size: 0.8em; color: grey;'> | |
| Powered by <a href="https://quantilytix.com" style='color: grey;'>Quantilytix</a> | © 2025 | |
| </div> | |
| """, | |
| unsafe_allow_html=True, | |
| ) | |
| if __name__ == "__main__": | |
| main() |