File size: 1,177 Bytes
325b94c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | from urllib.parse import urlparse
from schemas import SearchResult
def detect_source_type(url: str) -> str:
u = url.lower()
if u.endswith(".pdf"):
return "pdf"
if any(x in u for x in ["springer", "elsevier", "sciencedirect", "tandfonline"]):
return "journal"
if any(x in u for x in ["isbn", "books.google", "openlibrary"]):
return "book"
if any(x in u for x in [".edu", ".gov", "apa.org", "who.int"]):
return "institutional"
return "web"
BAD_DOMAINS = [
"facebook",
"youtube",
"tiktok",
"instagram",
"linkedin",
"twitter",
"blog",
"course",
"medium",
"quora",
"slideshare",
"wikipedia",
]
def is_good_domain(url: str) -> bool:
u = url.lower()
return not any(bad in u for bad in BAD_DOMAINS)
def extract_domain(url: str) -> str:
return urlparse(url).netloc.replace("www.", "")
def deduplicate_results(results: list[SearchResult]):
seen = set()
unique = []
for r in results:
key = (r.domain, r.title.lower().strip()[:120])
if key in seen:
continue
seen.add(key)
unique.append(r)
return unique
|