Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import pandas as pd | |
| import re | |
| from urllib.parse import urljoin | |
| HEADERS = {"User-Agent": "Mozilla/5.0"} | |
| # ------------------------------- | |
| # FETCH PAGE | |
| # ------------------------------- | |
| def fetch_page(url): | |
| try: | |
| response = requests.get(url, headers=HEADERS, timeout=10) | |
| return BeautifulSoup(response.text, "html.parser") | |
| except: | |
| return None | |
| # ------------------------------- | |
| # EXTRACT FROM BLOCKS (MAIN LOGIC) | |
| # ------------------------------- | |
| def extract_from_blocks(soup): | |
| blocks = soup.find_all(["p", "div", "li"]) | |
| data = [] | |
| for block in blocks: | |
| text = block.get_text(" ", strip=True) | |
| # Email detection | |
| email_match = re.search( | |
| r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text | |
| ) | |
| if email_match: | |
| email = email_match.group() | |
| # Name guess | |
| words = text.split() | |
| name = " ".join(words[:3]) if len(words) >= 3 else "Unknown" | |
| # Phone | |
| phone_match = re.search(r"\+?\d[\d\s\-]{8,15}", text) | |
| phone = phone_match.group() if phone_match else "Not Available" | |
| data.append([name, email, phone]) | |
| return data | |
| # ------------------------------- | |
| # TRY TO FIND PROFILE LINKS | |
| # ------------------------------- | |
| def get_profile_links(soup, base_url): | |
| links = set() | |
| for a in soup.find_all("a", href=True): | |
| href = a["href"] | |
| full = urljoin(base_url, href) | |
| keywords = ["faculty", "profile", "staff", "person", "people"] | |
| if any(k in full.lower() for k in keywords): | |
| links.add(full) | |
| return list(links) | |
| # ------------------------------- | |
| # EXTRACT FROM PROFILE PAGE | |
| # ------------------------------- | |
| def extract_from_profile(url): | |
| soup = fetch_page(url) | |
| if not soup: | |
| return None | |
| text = soup.get_text(" ", strip=True) | |
| # Name | |
| name = "Not Available" | |
| for tag in ["h1", "h2", "h3"]: | |
| t = soup.find(tag) | |
| if t: | |
| val = t.get_text(strip=True) | |
| if 2 < len(val) < 80: | |
| name = val | |
| break | |
| email_match = re.search( | |
| r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", text | |
| ) | |
| email = email_match.group() if email_match else "Not Available" | |
| # Phone | |
| phone_match = re.search(r"\+?\d[\d\s\-]{8,15}", text) | |
| phone = phone_match.group() if phone_match else "Not Available" | |
| return [name, email, phone] | |
| # ------------------------------- | |
| # MAIN SCRAPER FUNCTION | |
| # ------------------------------- | |
| def scrape(url): | |
| soup = fetch_page(url) | |
| if not soup: | |
| return None, "❌ Failed to load page" | |
| # Method 1: Direct extraction | |
| data = extract_from_blocks(soup) | |
| # Method 2: Profile-based extraction (fallback) | |
| if not data: | |
| links = get_profile_links(soup, url) | |
| for link in links[:20]: # limit for speed | |
| result = extract_from_profile(link) | |
| if result: | |
| data.append(result) | |
| if not data: | |
| return None, "❌ No data found (site may block scraping or use JavaScript)" | |
| df = pd.DataFrame(data, columns=["Name", "Email", "Phone"]) | |
| df = df.drop_duplicates() | |
| return df, f"✅ Found {len(df)} records" | |
| # ------------------------------- | |
| # STREAMLIT UI | |
| # ------------------------------- | |
| st.set_page_config(page_title="Faculty Scraper", layout="wide") | |
| st.title("🎓 Universal Faculty Scraper") | |
| st.markdown("Paste a college faculty page URL and extract emails, names, and phone numbers.") | |
| url = st.text_input("🔗 Enter Faculty Page URL") | |
| if st.button("🚀 Scrape"): | |
| if not url: | |
| st.warning("Please enter a URL") | |
| else: | |
| with st.spinner("Scraping in progress..."): | |
| df, msg = scrape(url) | |
| if df is not None: | |
| st.success(msg) | |
| st.dataframe(df, use_container_width=True) | |
| csv = df.to_csv(index=False).encode("utf-8") | |
| st.download_button( | |
| label="📥 Download CSV", | |
| data=csv, | |
| file_name="faculty_data.csv", | |
| mime="text/csv", | |
| ) | |
| else: | |
| st.error(msg) |