| from urllib.parse import urlparse |
| from schemas import SearchResult |
|
|
| def detect_source_type(url: str) -> str: |
| u = url.lower() |
|
|
| if u.endswith(".pdf"): |
| return "pdf" |
| if any(x in u for x in ["springer", "elsevier", "sciencedirect", "tandfonline"]): |
| return "journal" |
| if any(x in u for x in ["isbn", "books.google", "openlibrary"]): |
| return "book" |
| if any(x in u for x in [".edu", ".gov", "apa.org", "who.int"]): |
| return "institutional" |
| return "web" |
|
|
| BAD_DOMAINS = [ |
| "facebook", |
| "youtube", |
| "tiktok", |
| "instagram", |
| "linkedin", |
| "twitter", |
| "blog", |
| "course", |
| "medium", |
| "quora", |
| "slideshare", |
| "wikipedia", |
| ] |
|
|
| def is_good_domain(url: str) -> bool: |
| u = url.lower() |
| return not any(bad in u for bad in BAD_DOMAINS) |
|
|
|
|
| def extract_domain(url: str) -> str: |
| return urlparse(url).netloc.replace("www.", "") |
|
|
|
|
| def deduplicate_results(results: list[SearchResult]): |
| seen = set() |
| unique = [] |
|
|
| for r in results: |
| key = (r.domain, r.title.lower().strip()[:120]) |
| if key in seen: |
| continue |
| seen.add(key) |
| unique.append(r) |
|
|
| return unique |
|
|
