Spaces:
Runtime error
Runtime error
File size: 4,513 Bytes
43cd783 83dc7fc 43cd783 83dc7fc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | import streamlit as st
import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
st.set_page_config(page_title="Faculty Scraper", layout="wide")
st.title("🎓 Universal Faculty Scraper (Robust Mode)")
st.write("Handles even badly structured faculty pages.")
# -------- REGEX --------
EMAIL_REGEX = r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}"
PHONE_REGEX = r"\+?\d[\d\s\-]{8,15}"
BAD_NAMES = [
"welcome", "faculty", "central", "activities",
"department", "areas of interest", "important links"
]
# -------- NAME FROM EMAIL --------
def name_from_email(email):
name_part = email.split("@")[0]
name_part = name_part.replace(".", " ").replace("_", " ")
return " ".join([w.capitalize() for w in name_part.split()])
# -------- MAIN SCRAPER --------
def scrape_faculty(url):
headers = {"User-Agent": "Mozilla/5.0"}
try:
response = requests.get(url, headers=headers, verify=False, timeout=10)
except:
return pd.DataFrame(), "❌ Failed to load page"
soup = BeautifulSoup(response.text, "html.parser")
# -------- METHOD 1: BLOCK PARSING --------
blocks = soup.find_all(["div", "li", "tr", "section"])
raw_data = []
for block in blocks:
text = block.get_text(" ", strip=True)
if len(text) < 30:
continue
email_match = re.search(EMAIL_REGEX, text)
if not email_match:
continue
email = email_match.group()
phone_match = re.search(PHONE_REGEX, text)
phone = phone_match.group() if phone_match else "Not Available"
name = None
for tag in block.find_all(["h1", "h2", "h3", "h4", "strong", "b"]):
candidate = tag.get_text(strip=True)
if (
3 < len(candidate) < 60
and not re.search(EMAIL_REGEX, candidate)
and not any(b in candidate.lower() for b in BAD_NAMES)
):
name = candidate
break
raw_data.append([name, email, phone])
df = pd.DataFrame(raw_data, columns=["Name", "Email", "Phone"])
# -------- METHOD 2: FALLBACK (GLOBAL EXTRACTION) --------
if df.empty or len(df) < 5:
text = soup.get_text(" ", strip=True)
emails = re.findall(EMAIL_REGEX, text)
phones = re.findall(PHONE_REGEX, text)
emails = list(set(emails))
phones = list(set(phones))
data = []
for i, email in enumerate(emails):
name = name_from_email(email)
phone = phones[i] if i < len(phones) else "Not Available"
data.append([name, email, phone])
df = pd.DataFrame(data, columns=["Name", "Email", "Phone"])
# -------- CLEANING --------
cleaned_data = []
for email, group in df.groupby("Email"):
names = group["Name"].dropna().tolist()
phones = group["Phone"].dropna().tolist()
best_name = None
for n in names:
if n and not any(b in n.lower() for b in BAD_NAMES):
if any(t in n.lower() for t in ["dr", "mr", "ms", "prof"]):
best_name = n
break
if not best_name and names:
valid = [n for n in names if not any(b in str(n).lower() for b in BAD_NAMES)]
if valid:
best_name = max(valid, key=len)
if not best_name:
best_name = name_from_email(email)
best_phone = "Not Available"
for p in phones:
if p != "Not Available":
best_phone = p
break
cleaned_data.append([best_name, email, best_phone])
final_df = pd.DataFrame(cleaned_data, columns=["Name", "Email", "Phone"])
return final_df, "✅ Data Extracted Successfully"
# -------- UI --------
url = st.text_input("🔗 Enter Faculty Page URL")
if st.button("🚀 Scrape Data"):
if not url:
st.warning("Please enter a URL")
else:
with st.spinner("Scraping..."):
df, status = scrape_faculty(url)
st.success(status)
if not df.empty:
st.dataframe(df, use_container_width=True)
csv = df.to_csv(index=False).encode("utf-8")
st.download_button(
"📥 Download CSV",
csv,
"faculty_data.csv",
"text/csv"
)
else:
st.error("No data found.") |