Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import pandas as pd | |
| import re | |
| import time | |
| from selenium import webdriver | |
| from selenium.webdriver.chrome.options import Options | |
| from bs4 import BeautifulSoup | |
| st.set_page_config(page_title="Faculty Scraper", layout="wide") | |
| st.title("🎓 Universal Faculty Scraper (Clean + Smart)") | |
| EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}" | |
| # -------- DRIVER -------- | |
| def get_driver(): | |
| options = Options() | |
| options.binary_location = "/usr/bin/chromium" | |
| options.add_argument("--headless") | |
| options.add_argument("--no-sandbox") | |
| options.add_argument("--disable-dev-shm-usage") | |
| driver = webdriver.Chrome(options=options) | |
| return driver | |
| # -------- NAME CLEANING -------- | |
| def name_from_email(email): | |
| name_part = email.split("@")[0] | |
| # Replace separators | |
| name_part = re.sub(r"[._]", " ", name_part) | |
| # Split camel case | |
| name_part = re.sub(r'([a-z])([A-Z])', r'\1 \2', name_part) | |
| # Remove numbers | |
| name_part = re.sub(r'\d+', '', name_part) | |
| words = name_part.split() | |
| return " ".join([w.capitalize() for w in words if len(w) > 1]) | |
| # -------- PHONE CLEANING -------- | |
| def clean_phone(text): | |
| phones = re.findall(r"\+?\d[\d\s\-]{8,15}", text) | |
| for p in phones: | |
| p_clean = p.replace(" ", "") | |
| # Reject year-like values | |
| if re.search(r"\d{4}-\d{2}", p_clean): | |
| continue | |
| digits = re.sub(r"\D", "", p_clean) | |
| # Accept only realistic phone lengths | |
| if 10 <= len(digits) <= 13: | |
| return p_clean | |
| return "Not Available" | |
| # -------- MAIN SCRAPER -------- | |
| def scrape_faculty(url): | |
| driver = get_driver() | |
| driver.get(url) | |
| time.sleep(5) | |
| soup = BeautifulSoup(driver.page_source, "html.parser") | |
| driver.quit() | |
| data = [] | |
| blocks = soup.find_all(["tr", "div", "li", "section"]) | |
| for block in blocks: | |
| text = block.get_text(" ", strip=True) | |
| if len(text) < 20: | |
| continue | |
| # -------- EMAIL -------- | |
| email_match = re.search(EMAIL_REGEX, text) | |
| if not email_match: | |
| mail = block.find("a", href=re.compile("mailto:", re.I)) | |
| if mail: | |
| email = mail.get("href").replace("mailto:", "").strip() | |
| else: | |
| continue | |
| else: | |
| email = email_match.group() | |
| # -------- NAME -------- | |
| name = None | |
| # Try headings | |
| h = block.find(["h1", "h2", "h3", "h4", "strong", "b"]) | |
| if h: | |
| name = h.get_text(strip=True) | |
| # Fallback | |
| if not name or len(name) < 3: | |
| name = name_from_email(email) | |
| # -------- PHONE -------- | |
| phone = clean_phone(text) | |
| data.append([name, email, phone]) | |
| df = pd.DataFrame(data, columns=["Name", "Email", "Phone"]) | |
| # -------- REMOVE DUPLICATES -------- | |
| df = df.drop_duplicates(subset=["Email"]) | |
| return df | |
| # -------- UI -------- | |
| url = st.text_input("🔗 Enter Faculty Page URL") | |
| if st.button("🚀 Scrape Data"): | |
| if not url: | |
| st.warning("Please enter a URL") | |
| else: | |
| with st.spinner("Scraping intelligently..."): | |
| df = scrape_faculty(url) | |
| if not df.empty: | |
| st.success("✅ Data Extracted Successfully") | |
| st.dataframe(df, use_container_width=True) | |
| csv = df.to_csv(index=False).encode("utf-8") | |
| st.download_button( | |
| "📥 Download CSV", | |
| csv, | |
| "faculty_data.csv", | |
| "text/csv" | |
| ) | |
| else: | |
| st.error("❌ No data found") |