Spaces:
Runtime error
Runtime error
| import streamlit as st | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import pandas as pd | |
| import re | |
| from urllib.parse import urljoin, urlparse | |
| import time | |
| # --- CONFIG & CONSTANTS --- | |
| USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" | |
| HEADERS = {"User-Agent": USER_AGENT} | |
| EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' | |
| # Improved Phone Regex to avoid matching years/dates | |
| PHONE_REGEX = r'(?:\+?\d{1,3}[-.\s]?)?\(?\d{3,4}\)?[-.\s]?\d{3,4}[-.\s]?\d{3,4}' | |
| TITLES = ["Dr", "Prof", "Mr", "Ms", "Mrs", "Shri", "Smt", "Doctor"] | |
| DESIGNATIONS = [ | |
| "Assistant Professor", "Associate Professor", "Professor", | |
| "HOD", "Head of Department", "Lecturer", "Dean", | |
| "Director", "Faculty", "Instructor", "Registrar" | |
| ] | |
| # Keywords that invalidate a string as a Name | |
| INVALID_NAME_TOKENS = [ | |
| "phone", "email", "contact", "dept", "department", "office", "fax", | |
| "mobile", "university", "college", "address", "profile", "research", | |
| "publication", "student", "grievance", "cell", "ext", "tel", "call" | |
| ] | |
| # --- UTILITY FUNCTIONS --- | |
| def clean_text(text): | |
| if not text: return "" | |
| # Remove non-breaking spaces and extra whitespace | |
| text = text.replace('\xa0', ' ') | |
| return re.sub(r'\s+', ' ', text).strip() | |
| def is_valid_name(name_str): | |
| if not name_str: return False | |
| clean = clean_text(name_str) | |
| # 1. Length constraint | |
| if not (3 <= len(clean) <= 60): return False | |
| # 2. Reject if contains digits | |
| if any(char.isdigit() for char in clean): return False | |
| # 3. Reject if contains invalid keywords (case-insensitive) | |
| lower_clean = clean.lower() | |
| if any(token in lower_clean for token in INVALID_NAME_TOKENS): return False | |
| # 4. Must contain at least one alphabet | |
| if not re.search(r'[a-zA-Z]', clean): return False | |
| # 5. Must have at least 2 words OR a known title | |
| words = clean.split() | |
| has_title = any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES) | |
| if len(words) >= 2 or has_title: | |
| return True | |
| return False | |
| def score_name(name, base_confidence): | |
| if not name: return 0 | |
| score = float(base_confidence) | |
| words = name.split() | |
| # Bonus for academic titles | |
| if any(t.lower() in [w.lower().strip('.') for w in words] for t in TITLES): | |
| score += 1.5 | |
| # Bonus for standard name lengths (2-3 words) | |
| if 2 <= len(words) <= 3: | |
| score += 1.0 | |
| # Penalty for very long strings | |
| if len(name) > 40: | |
| score -= 0.5 | |
| return score | |
| def extract_designation(text): | |
| text_lower = text.lower() | |
| # Sort designations by length descending to match "Assistant Professor" before "Professor" | |
| for desig in sorted(DESIGNATIONS, key=len, reverse=True): | |
| if desig.lower() in text_lower: | |
| return desig | |
| return "" | |
| # --- SCRAPER ENGINE --- | |
| class FacultyScraper: | |
| def __init__(self): | |
| self.visited_urls = set() | |
| self.candidates = [] # List of dicts: {email, name, phone, designation, score} | |
| def fetch_soup(self, url): | |
| try: | |
| response = requests.get(url, headers=HEADERS, timeout=15, verify=False) | |
| response.raise_for_status() | |
| return BeautifulSoup(response.text, 'html.parser') | |
| except Exception as e: | |
| return None | |
| def extract_from_soup(self, soup): | |
| if not soup: return | |
| # Strategy A: Block-based (High Confidence) | |
| # Look for containers that likely hold one person's info | |
| for block_tag in ['div', 'tr', 'li', 'section', 'article']: | |
| for block in soup.find_all(block_tag): | |
| # Only process blocks that contain an email | |
| block_text = block.get_text(" ", strip=True) | |
| emails = re.findall(EMAIL_REGEX, block_text) | |
| if not emails: continue | |
| email = emails[0].lower() | |
| # Find name: look for headings or bold text within this specific block | |
| name = "" | |
| for header in block.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'strong', 'b']): | |
| possible = clean_text(header.get_text()) | |
| if is_valid_name(possible): | |
| name = possible | |
| break | |
| phone_match = re.search(PHONE_REGEX, block_text) | |
| phone = phone_match.group(0) if phone_match else "" | |
| desig = extract_designation(block_text) | |
| if name: | |
| self.candidates.append({ | |
| "Email": email, "Name": name, "Phone": phone, | |
| "Designation": desig, "Score": score_name(name, 3) | |
| }) | |
| # Strategy B: Context-based (Medium Confidence) | |
| full_text = soup.get_text(" ", strip=True) | |
| for match in re.finditer(EMAIL_REGEX, full_text): | |
| email = match.group(0).lower() | |
| # Extract window around email | |
| start = max(0, match.start() - 150) | |
| end = min(len(full_text), match.end() + 150) | |
| context = full_text[start:end] | |
| # Logic: Name usually appears BEFORE the email in text | |
| pre_context = full_text[max(0, match.start()-100) : match.start()] | |
| lines = [l.strip() for l in pre_context.split(' ') if l.strip()] | |
| best_context_name = "" | |
| for line in reversed(lines): | |
| if is_valid_name(line): | |
| best_context_name = line | |
| break | |
| if best_context_name: | |
| self.candidates.append({ | |
| "Email": email, "Name": best_context_name, | |
| "Phone": re.search(PHONE_REGEX, context).group(0) if re.search(PHONE_REGEX, context) else "", | |
| "Designation": extract_designation(context), | |
| "Score": score_name(best_context_name, 2) | |
| }) | |
| # Strategy C: Fallback (Low Confidence) | |
| all_emails = re.findall(EMAIL_REGEX, full_text) | |
| for email in all_emails: | |
| email = email.lower() | |
| username = email.split('@')[0] | |
| fallback_name = username.replace('.', ' ').replace('_', ' ').title() | |
| if len(fallback_name) > 3: | |
| self.candidates.append({ | |
| "Email": email, "Name": fallback_name, "Phone": "", | |
| "Designation": "", "Score": 1.0 | |
| }) | |
| def run(self, start_url): | |
| # 1. Crawl Logic | |
| self.visited_urls.add(start_url) | |
| soup = self.fetch_soup(start_url) | |
| if not soup: return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"]) | |
| self.extract_from_soup(soup) | |
| # Follow links (Max 5 pages total) | |
| links = soup.find_all('a', href=True) | |
| crawled_count = 1 | |
| for link in links: | |
| if crawled_count >= 5: break | |
| href = urljoin(start_url, link['href']) | |
| parsed = urlparse(href) | |
| # Stay on same domain, look for keywords | |
| if parsed.netloc == urlparse(start_url).netloc: | |
| if any(k in href.lower() for k in ["faculty", "staff", "profile", "people", "directory"]): | |
| if href not in self.visited_urls: | |
| self.visited_urls.add(href) | |
| sub_soup = self.fetch_soup(href) | |
| if sub_soup: | |
| self.extract_from_soup(sub_soup) | |
| crawled_count += 1 | |
| # 2. Process Candidates | |
| if not self.candidates: | |
| return pd.DataFrame(columns=["Name", "Email", "Phone", "Designation"]) | |
| df = pd.DataFrame(self.candidates) | |
| # 3. Aggressive Deduplication and Selection | |
| # Group by Email and pick the candidate with the highest score | |
| df = df.sort_values(by="Score", ascending=False) | |
| # Clean specific columns before final drop | |
| df['Name'] = df['Name'].apply(clean_text) | |
| df['Phone'] = df['Phone'].apply(clean_text) | |
| df['Designation'] = df['Designation'].apply(clean_text) | |
| # Remove records where Email is clearly invalid (optional but safer) | |
| df = df[df['Email'].str.contains(r'[^@]+@[^@]+\.[^@]+')] | |
| # Final unique selection per Email | |
| df_final = df.groupby('Email').first().reset_index() | |
| # Final column ordering | |
| return df_final[["Name", "Email", "Phone", "Designation"]] | |
| # --- STREAMLIT UI --- | |
| def main(): | |
| st.set_page_config(page_title="Faculty Scraper Pro", layout="wide") | |
| st.title("🏛️ Production-Grade Faculty Extractor") | |
| st.markdown("Extracts clean, deduplicated faculty data using multi-strategy block analysis.") | |
| url = st.text_input("Enter Faculty Directory URL:", placeholder="https://www.university.edu/science/faculty") | |
| if st.button("Start Extraction", type="primary"): | |
| if not url: | |
| st.error("Please enter a URL first.") | |
| return | |
| with st.spinner("Analyzing pages and resolving duplicates..."): | |
| scraper = FacultyScraper() | |
| results_df = scraper.run(url) | |
| if not results_df.empty: | |
| st.success(f"Extracted {len(results_df)} unique faculty members.") | |
| # Cleanup display: Ensure no weird characters or messed up rows | |
| results_df = results_df.fillna("").astype(str) | |
| st.dataframe(results_df, use_container_width=True) | |
| csv = results_df.to_csv(index=False).encode('utf-8') | |
| st.download_button( | |
| label="Download Data as CSV", | |
| data=csv, | |
| file_name="faculty_data.csv", | |
| mime="text/csv", | |
| ) | |
| else: | |
| st.warning("No faculty data found. Ensure the URL is a public directory page.") | |
| st.divider() | |
| st.caption("Built with Python, BeautifulSoup, and pandas. Designed for production-grade robustness.") | |
| if __name__ == "__main__": | |
| # Disable warnings for self-signed certificates common in edu sites | |
| import urllib3 | |
| urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) | |
| main() |