Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import pandas as pd | |
| import re | |
| import urllib3 | |
| urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) | |
| st.set_page_config(page_title="Faculty Scraper", layout="wide") | |
| st.title("🎓 Universal Faculty Scraper (Robust Mode)") | |
| st.write("Handles even badly structured faculty pages.") | |
| # -------- REGEX -------- | |
| EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}" | |
| PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}" | |
| BAD_NAMES = [ | |
| "welcome", "faculty", "central", "activities", | |
| "department", "areas of interest", "important links" | |
| ] | |
| # -------- NAME FROM EMAIL -------- | |
| def name_from_email(email): | |
| name_part = email.split("@")[0] | |
| name_part = name_part.replace(".", " ").replace("_", " ") | |
| return " ".join([w.capitalize() for w in name_part.split()]) | |
| # -------- MAIN SCRAPER -------- | |
| def scrape_faculty(url): | |
| headers = {"User-Agent": "Mozilla/5.0"} | |
| try: | |
| response = requests.get(url, headers=headers, verify=False, timeout=10) | |
| except: | |
| return pd.DataFrame(), "❌ Failed to load page" | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # -------- METHOD 1: BLOCK PARSING -------- | |
| blocks = soup.find_all(["div", "li", "tr", "section"]) | |
| raw_data = [] | |
| for block in blocks: | |
| text = block.get_text(" ", strip=True) | |
| if len(text) < 30: | |
| continue | |
| email_match = re.search(EMAIL_REGEX, text) | |
| if not email_match: | |
| continue | |
| email = email_match.group() | |
| phone_match = re.search(PHONE_REGEX, text) | |
| phone = phone_match.group() if phone_match else "Not Available" | |
| name = None | |
| for tag in block.find_all(["h1", "h2", "h3", "h4", "strong", "b"]): | |
| candidate = tag.get_text(strip=True) | |
| if ( | |
| 3 < len(candidate) < 60 | |
| and not re.search(EMAIL_REGEX, candidate) | |
| and not any(b in candidate.lower() for b in BAD_NAMES) | |
| ): | |
| name = candidate | |
| break | |
| raw_data.append([name, email, phone]) | |
| df = pd.DataFrame(raw_data, columns=["Name", "Email", "Phone"]) | |
| # -------- METHOD 2: FALLBACK (GLOBAL EXTRACTION) -------- | |
| if df.empty or len(df) < 5: | |
| text = soup.get_text(" ", strip=True) | |
| emails = re.findall(EMAIL_REGEX, text) | |
| phones = re.findall(PHONE_REGEX, text) | |
| emails = list(set(emails)) | |
| phones = list(set(phones)) | |
| data = [] | |
| for i, email in enumerate(emails): | |
| name = name_from_email(email) | |
| phone = phones[i] if i < len(phones) else "Not Available" | |
| data.append([name, email, phone]) | |
| df = pd.DataFrame(data, columns=["Name", "Email", "Phone"]) | |
| # -------- CLEANING -------- | |
| cleaned_data = [] | |
| for email, group in df.groupby("Email"): | |
| names = group["Name"].dropna().tolist() | |
| phones = group["Phone"].dropna().tolist() | |
| best_name = None | |
| for n in names: | |
| if n and not any(b in n.lower() for b in BAD_NAMES): | |
| if any(t in n.lower() for t in ["dr", "mr", "ms", "prof"]): | |
| best_name = n | |
| break | |
| if not best_name and names: | |
| valid = [n for n in names if not any(b in str(n).lower() for b in BAD_NAMES)] | |
| if valid: | |
| best_name = max(valid, key=len) | |
| if not best_name: | |
| best_name = name_from_email(email) | |
| best_phone = "Not Available" | |
| for p in phones: | |
| if p != "Not Available": | |
| best_phone = p | |
| break | |
| cleaned_data.append([best_name, email, best_phone]) | |
| final_df = pd.DataFrame(cleaned_data, columns=["Name", "Email", "Phone"]) | |
| return final_df, "✅ Data Extracted Successfully" | |
| # -------- UI -------- | |
| url = st.text_input("🔗 Enter Faculty Page URL") | |
| if st.button("🚀 Scrape Data"): | |
| if not url: | |
| st.warning("Please enter a URL") | |
| else: | |
| with st.spinner("Scraping..."): | |
| df, status = scrape_faculty(url) | |
| st.success(status) | |
| if not df.empty: | |
| st.dataframe(df, use_container_width=True) | |
| csv = df.to_csv(index=False).encode("utf-8") | |
| st.download_button( | |
| "📥 Download CSV", | |
| csv, | |
| "faculty_data.csv", | |
| "text/csv" | |
| ) | |
| else: | |
| st.error("No data found.") |