import io from collections import Counter from urllib.parse import urljoin, urlparse import gradio as gr import httpx import pdfplumber from bs4 import BeautifulSoup from docx import Document # --------------------------------------------------------------------------- # Model — imported from api.py so we share the single loaded instance. # api.py populates _model lazily in its background thread; app.py references # it through the module so it always sees the latest value. # --------------------------------------------------------------------------- import api as _api_module _HTTP_HEADERS = { "User-Agent": ( "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def extract_text_from_file(file_path: str) -> str: """Extract plain text from a TXT, PDF, or DOCX file.""" if file_path.endswith(".pdf"): with pdfplumber.open(file_path) as pdf: return "\n".join(page.extract_text() or "" for page in pdf.pages) elif file_path.endswith(".docx"): doc = Document(file_path) return "\n".join(p.text for p in doc.paragraphs) else: with open(file_path, "r", errors="ignore") as f: return f.read() def _get_base(url: str) -> str: p = urlparse(url) return f"{p.scheme}://{p.netloc}" def _abs(href: str, base: str, page_url: str) -> str: if href.startswith("//"): scheme = urlparse(page_url).scheme return f"{scheme}:{href}" return urljoin(page_url, href) def _collect_sibling_content(heading_tag) -> tuple[str, str]: """Walk siblings of a heading tag until the next heading of same/higher level. Returns (description_text, first_external_link_href).""" level = int(heading_tag.name[1]) stop_tags = {f"h{i}" for i in range(1, level + 1)} parts, link = [], "" node = heading_tag.next_sibling while node: name = getattr(node, "name", None) if name in stop_tags: break if name: text = node.get_text(" ", strip=True) if text: parts.append(text) if not link: a = node.find("a", href=True) if hasattr(node, "find") else None if a and len(a.get_text(strip=True)) > 2: link = a["href"] node = node.next_sibling return " ".join(parts)[:600], link def scrape_items(url: str) -> list[dict]: """ Generic scraper using httpx (no browser required). Tries multiple strategies to extract individual items from any page. """ try: with httpx.Client(headers=_HTTP_HEADERS, follow_redirects=True, timeout=20.0) as client: resp = client.get(url) resp.raise_for_status() html = resp.text except Exception as e: return [{"title": url, "link": url, "description": f"[Error: {e}]"}] soup = BeautifulSoup(html, "html.parser") base = _get_base(url) # Strip global noise for tag in soup(["script", "style", "nav", "footer", "header", "noscript"]): tag.decompose() items = [] # ── Strategy 1: Repeated card containers (article, li, div with consistent class) ── from collections import Counter candidate_classes: Counter = Counter() for el in soup.find_all(["article", "li", "div"], class_=True): for cls in el.get("class", []): # Skip utility/layout/generic classes if any(skip in cls.lower() for skip in [ "footer", "nav", "menu", "modal", "cookie", "banner", "wrapper", "container", "row", "col", "icon", "clearfix", "active", "hidden", "visible", "block", "item", "list", ]): continue candidate_classes[cls] += 1 # Pick classes that repeat ≥3 times (list of cards) card_classes = [cls for cls, count in candidate_classes.most_common(5) if count >= 3] for cls in card_classes: cards = soup.find_all(["article", "li", "div"], class_=cls) if len(cards) < 3: continue batch = [] for card in cards: heading = card.find(["h1", "h2", "h3", "h4", "h5"]) if not heading: continue title = heading.get_text(" ", strip=True).strip() if len(title) < 5: continue # Find best link in card a = heading.find("a", href=True) or card.find("a", href=True) link = _abs(a["href"], base, url) if a else url desc = card.get_text(" ", strip=True) batch.append({"title": title, "link": link, "description": desc}) if len(batch) >= 3: items = batch break # ── Strategy 2: Heading-per-item pattern (h2/h3 each = one scholarship/programme) ── if not items: main = soup.find("main") or soup.find("div", id=lambda x: x and "content" in x.lower()) or soup.body for heading_tag in ["h3", "h2", "h4"]: headings = main.find_all(heading_tag) if main else [] if len(headings) < 3: continue batch = [] for h in headings: title = h.get_text(" ", strip=True).strip() if len(title) < 5: continue # Skip nav/footer headings if any(word in title.lower() for word in ["information for", "quick links", "contact us", "follow us", "about ucl", "social media"]): continue desc, sibling_link = _collect_sibling_content(h) # Prefer anchor on the heading itself, then sibling link h_id = h.get("id") or (h.find("a") and h.find("a").get("id")) if h_id: link = f"{url.split('#')[0]}#{h_id}" elif sibling_link: link = _abs(sibling_link, base, url) else: a = h.find("a", href=True) link = _abs(a["href"], base, url) if a else url batch.append({"title": title, "link": link, "description": f"{title}. {desc}"}) if len(batch) >= 3: items = batch break # ── Strategy 3: Named anchor links in a table-of-contents section ── if not items: seen = set() for a in soup.find_all("a", href=True): href = a["href"] title = a.get_text(" ", strip=True) if len(title) < 8 or href in seen: continue seen.add(href) link = _abs(href, base, url) parent = a.find_parent(["li", "p", "td", "div"]) desc = parent.get_text(" ", strip=True) if parent else title items.append({"title": title, "link": link, "description": desc}) # ── Fallback: overlapping text chunks of entire page ── if not items: text = soup.get_text(" ", strip=True) words = text.split() for i in range(0, len(words), 450): chunk = " ".join(words[i : i + 500]) items.append({"title": f"Section {i//450+1}", "link": url, "description": chunk}) return items import numpy as np import faiss def retrieve_top_items(user_text: str, items: list[dict], top_k: int = 5) -> list[dict]: """Embed user text and items using the shared model, rank with FAISS.""" if not items: return [] model = _api_module._model if model is None: return [] texts = [it["description"] for it in items] user_emb = model.encode([user_text], convert_to_numpy=True).astype("float32") item_embs = model.encode(texts, convert_to_numpy=True, show_progress_bar=False).astype("float32") idx = faiss.IndexFlatL2(item_embs.shape[1]) idx.add(item_embs) _, indices = idx.search(user_emb, min(top_k, len(items))) seen_links: set[str] = set() results = [] for i in indices[0]: it = items[i] if it["link"] not in seen_links: seen_links.add(it["link"]) results.append(it) return results # --------------------------------------------------------------------------- # Tab 1: Quick Match — uses the pre-loaded seed index via /match endpoint # --------------------------------------------------------------------------- def _extract_profile_text(profile_file) -> str: if profile_file is None: return "" return extract_text_from_file(profile_file) def quick_match(profile_file, profile_text: str, top_k: int) -> str: """Match against the pre-loaded seed index (instant — no scraping).""" text = profile_text.strip() if not text and profile_file is not None: text = _extract_profile_text(profile_file) if not text: return "⚠️ Please upload a resume/CV file or type your profile in the text box." if not _api_module._index_ready.is_set(): return "⏳ The scholarship index is still loading. This usually takes 10–30 seconds. Please try again shortly." # Call the match logic directly (in-process, no HTTP round-trip) from api import MatchRequest, match as _match try: resp = _match(MatchRequest(profile=text, top_k=int(top_k))) except Exception as e: return f"❌ Error: {e}" if not resp.results: return "No matching scholarships found. Try broadening your profile description." lines = [f"### 🎓 Top {len(resp.results)} Scholarships for Your Profile\n", f"*Searched {resp.total_indexed} indexed scholarships*\n\n---\n"] for i, r in enumerate(resp.results, 1): lines.append(f"**{i}. [{r.title}]({r.link})**") if r.source: lines.append(f" *Source: {r.source}*") lines.append(f" {r.description[:250]}…\n") return "\n".join(lines) # --------------------------------------------------------------------------- # Tab 2: Custom Search — user provides URLs, scrapes on-the-fly with httpx # --------------------------------------------------------------------------- def custom_search(profile_file, profile_text: str, urls_input: str, top_k: int) -> str: text = profile_text.strip() if not text and profile_file is not None: text = _extract_profile_text(profile_file) if not text: return "⚠️ Please upload a resume/CV file or type your profile in the text box." if not urls_input.strip(): return "⚠️ Please enter at least one URL to search." urls = [u.strip() for u in urls_input.split(",") if u.strip()] all_items: list[dict] = [] log_lines: list[str] = [] for url in urls: batch = scrape_items(url) valid = [b for b in batch if not b["description"].startswith("[Error")] all_items.extend(valid) log_lines.append(f"- `{url}` → {len(valid)} items") if not all_items: return "❌ No content could be scraped from the provided URLs. They may block bots or require JavaScript." top_items = retrieve_top_items(text, all_items, top_k=int(top_k)) if not top_items: return "No matching items found." lines = [f"### 🔍 Top {len(top_items)} Matches from Custom URLs\n", "**Scrape log:**\n" + "\n".join(log_lines) + "\n\n---\n"] for i, item in enumerate(top_items, 1): lines.append(f"**{i}. [{item['title']}]({item['link']})**") lines.append(f" {item['description'][:250]}…\n") return "\n".join(lines) # --------------------------------------------------------------------------- # Gradio UI — two tabs # --------------------------------------------------------------------------- _PROFILE_HELP = "Upload a PDF/DOCX resume **or** type/paste your background below." with gr.Blocks( title="Fundora — Scholarship Matcher", theme=gr.themes.Soft(), css=".gr-button-primary { background: #2563eb !important; }", ) as demo: gr.Markdown( "# 🎓 Fundora — AI Scholarship Matcher\n" "Find scholarships that match your profile from a curated index of 90+ " "global programmes (Chevening, DAAD, Fulbright, Google PhD, MEXT, Vanier, and more).\n\n" "> **Free to use. No login required.**" ) with gr.Tabs(): # ── Tab 1: Quick Match (seed index) ─────────────────────────────── with gr.TabItem("⚡ Quick Match (recommended)"): gr.Markdown( "Match against our **pre-loaded index of 90+ scholarships** — results in seconds." ) with gr.Row(): with gr.Column(scale=1): qm_file = gr.File( label="Upload Resume / CV (PDF, DOCX, TXT)", file_types=[".txt", ".pdf", ".docx"], ) qm_text = gr.Textbox( label="Or paste / type your profile here", placeholder=( "e.g. Indian student, BE Electronics, GPA 8.0, " "ML research, LLM quantization, seeking MS abroad…" ), lines=5, ) qm_topk = gr.Slider(1, 20, value=8, step=1, label="Results to show") qm_btn = gr.Button("Find Scholarships", variant="primary") with gr.Column(scale=2): qm_out = gr.Markdown(label="Results") qm_btn.click( quick_match, inputs=[qm_file, qm_text, qm_topk], outputs=qm_out, ) # ── Tab 2: Custom Search (user-provided URLs) ────────────────────── with gr.TabItem("🔍 Custom Search (any URL)"): gr.Markdown( "Paste **any scholarship or opportunity page URLs** — Fundora will scrape " "and rank them against your profile on the fly." ) with gr.Row(): with gr.Column(scale=1): cs_file = gr.File( label="Upload Resume / CV (PDF, DOCX, TXT)", file_types=[".txt", ".pdf", ".docx"], ) cs_text = gr.Textbox( label="Or paste / type your profile here", placeholder="e.g. Nigerian student, BSc Computer Science, 3.8 GPA, data science focus…", lines=5, ) cs_urls = gr.Textbox( label="URLs to search (comma-separated)", placeholder="https://www.daad.de/..., https://opportunitydesk.org/...", lines=3, ) cs_topk = gr.Slider(1, 20, value=5, step=1, label="Results to show") cs_btn = gr.Button("Search", variant="primary") with gr.Column(scale=2): cs_out = gr.Markdown(label="Results") cs_btn.click( custom_search, inputs=[cs_file, cs_text, cs_urls, cs_topk], outputs=cs_out, ) gr.Markdown( "---\n" "Made with ❤️ by [Kabir Potdar](https://github.com/Kabir08) · " "[GitHub](https://github.com/Kabir08/Fundora) · " "[API docs](/docs) · [Privacy](/privacy)" ) if __name__ == "__main__": demo.launch(server_port=7860)