import streamlit as st import requests from bs4 import BeautifulSoup import pandas as pd import re from urllib.parse import urljoin, urlparse import time # --- CONFIG & CONSTANTS --- USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" HEADERS = {"User-Agent": USER_AGENT} EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' # Improved Phone Regex to avoid matching years/dates PHONE_REGEX = r'(?:\+?\d{1,3}[-.\s]?)?\(?\d{3,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}' TITLES = ["Dr", "Prof", "Mr", "Ms", "Mrs", "Shri", "Smt", "Doctor"] DESIGNATIONS = [ "Assistant Professor", "Associate Professor", "Professor", "HOD", "Head of Department", "Lecturer", "Dean", "Director", "Faculty", "Instructor", "Registrar" ] # Keywords that invalidate a string as a Name INVALID_NAME_TOKENS = [ "phone", "email", "contact", "dept", "department", "office", "fax", "mobile", "university", "college", "address", "profile", "research", "publication", "student", "grievance", "cell", "ext", "tel", "call" ] # --- UTILITY FUNCTIONS --- def clean_text(text): if not text: return "" # Remove non-breaking spaces and extra whitespace text = text.replace('\xa0', ' ') return re.sub(r'\s+', ' ', text).strip() def is_valid_name(name_str): if not name_str: return False clean = clean_text(name_str) # 1. Length constraint if not (3 <= len(clean) <= 60): return False # 2. Reject if contains digits if any(char.isdigit() for char in clean): return False # 3. Reject if contains invalid keywords (case-insensitive) lower_clean = clean.lower() if any(token in lower_clean for token in INVALID_NAME_TOKENS): return False # 4. Must contain at least one alphabet if not re.search(r'[a-zA-Z]', clean): return False # 5. Must have at least 2 words OR a known title words = clean.split() has_title = any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES) if len(words) >= 2 or has_title: return True return False def score_name(name, base_confidence): if not name: return 0 score = float(base_confidence) words = name.split() # Bonus for academic titles if any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES): score += 1.5 # Bonus for standard name lengths (2-3 words) if 2 <= len(words) <= 3: score += 1.0 # Penalty for very long strings if len(name) > 40: score -= 0.5 return score def extract_designation(text): text_lower = text.lower() # Sort designations by length descending to match "Assistant Professor" before "Professor" for desig in sorted(DESIGNATIONS, key=len, reverse=True): if desig.lower() in text_lower: return desig return "" # --- SCRAPER ENGINE --- class FacultyScraper: def __init__(self): self.visited_urls = set() self.candidates = [] # List of dicts: {email, name, phone, designation, score} def fetch_soup(self, url): try: response = requests.get(url, headers=HEADERS, timeout=15, verify=False) response.raise_for_status() return BeautifulSoup(response.text, 'html.parser') except Exception as e: return None def extract_from_soup(self, soup): if not soup: return # Strategy A: Block-based (High Confidence) # Look for containers that likely hold one person's info for block_tag in ['div', 'tr', 'li', 'section', 'article']: for block in soup.find_all(block_tag): # Only process blocks that contain an email block_text = block.get_text(" ", strip=True) emails = re.findall(EMAIL_REGEX, block_text) if not emails: continue email = emails[0].lower() # Find name: look for headings or bold text within this specific block name = "" for header in block.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'strong', 'b']): possible = clean_text(header.get_text()) if is_valid_name(possible): name = possible break phone_match = re.search(PHONE_REGEX, block_text) phone = phone_match.group(0) if phone_match else "" desig = extract_designation(block_text) if name: self.candidates.append({ "Email": email, "Name": name, "Phone": phone, "Designation": desig, "Score": score_name(name, 3) }) # Strategy B: Context-based (Medium Confidence) full_text = soup.get_text(" ", strip=True) for match in re.finditer(EMAIL_REGEX, full_text): email = match.group(0).lower() # Extract window around email start = max(0, match.start() - 150) end = min(len(full_text), match.end() + 150) context = full_text[start:end] # Logic: Name usually appears BEFORE the email in text pre_context = full_text[max(0, match.start()-100) : match.start()] lines = [l.strip() for l in pre_context.split(' ') if l.strip()] best_context_name = "" for line in reversed(lines): if is_valid_name(line): best_context_name = line break if best_context_name: self.candidates.append({ "Email": email, "Name": best_context_name, "Phone": re.search(PHONE_REGEX, context).group(0) if re.search(PHONE_REGEX, context) else "", "Designation": extract_designation(context), "Score": score_name(best_context_name, 2) }) # Strategy C: Fallback (Low Confidence) all_emails = re.findall(EMAIL_REGEX, full_text) for email in all_emails: email = email.lower() username = email.split('@')[0] fallback_name = username.replace('.', ' ').replace('_', ' ').title() if len(fallback_name) > 3: self.candidates.append({ "Email": email, "Name": fallback_name, "Phone": "", "Designation": "", "Score": 1.0 }) def run(self, start_url): # 1. Crawl Logic self.visited_urls.add(start_url) soup = self.fetch_soup(start_url) if not soup: return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"]) self.extract_from_soup(soup) # Follow links (Max 5 pages total) links = soup.find_all('a', href=True) crawled_count = 1 for link in links: if crawled_count >= 5: break href = urljoin(start_url, link['href']) parsed = urlparse(href) # Stay on same domain, look for keywords if parsed.netloc == urlparse(start_url).netloc: if any(k in href.lower() for k in ["faculty", "staff", "profile", "people", "directory"]): if href not in self.visited_urls: self.visited_urls.add(href) sub_soup = self.fetch_soup(href) if sub_soup: self.extract_from_soup(sub_soup) crawled_count += 1 # 2. Process Candidates if not self.candidates: return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"]) df = pd.DataFrame(self.candidates) # 3. Aggressive Deduplication and Selection # Group by Email and pick the candidate with the highest score df = df.sort_values(by="Score", ascending=False) # Clean specific columns before final drop df['Name'] = df['Name'].apply(clean_text) df['Phone'] = df['Phone'].apply(clean_text) df['Designation'] = df['Designation'].apply(clean_text) # Remove records where Email is clearly invalid (optional but safer) df = df[df['Email'].str.contains(r'[^@]+@[^@]+\.[^@]+')] # Final unique selection per Email df_final = df.groupby('Email').first().reset_index() # Final column ordering return df_final[["Name", "Email", "Phone", "Designation"]] # --- STREAMLIT UI --- def main(): st.set_page_config(page_title="Faculty Scraper Pro", layout="wide") st.title("🏛️ Production-Grade Faculty Extractor") st.markdown("Extracts clean, deduplicated faculty data using multi-strategy block analysis.") url = st.text_input("Enter Faculty Directory URL:", placeholder="https://www.university.edu/science/faculty") if st.button("Start Extraction", type="primary"): if not url: st.error("Please enter a URL first.") return with st.spinner("Analyzing pages and resolving duplicates..."): scraper = FacultyScraper() results_df = scraper.run(url) if not results_df.empty: st.success(f"Extracted {len(results_df)} unique faculty members.") # Cleanup display: Ensure no weird characters or messed up rows results_df = results_df.fillna("").astype(str) st.dataframe(results_df, use_container_width=True) csv = results_df.to_csv(index=False).encode('utf-8') st.download_button( label="Download Data as CSV", data=csv, file_name="faculty_data.csv", mime="text/csv", ) else: st.warning("No faculty data found. Ensure the URL is a public directory page.") st.divider() st.caption("Built with Python, BeautifulSoup, and pandas. Designed for production-grade robustness.") if __name__ == "__main__": # Disable warnings for self-signed certificates common in edu sites import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) main()