| |
| import urllib.parse |
| import re |
| import constants |
|
|
| def normalize_url(url: str) -> str: |
| parsed = urllib.parse.urlparse(url) |
| clean = parsed.scheme + "://" + parsed.netloc + parsed.path |
| return clean.rstrip("/") |
|
|
| def extract_real_video_url(url: str) -> str: |
| if not url: return "" |
| parsed_url = urllib.parse.urlparse(url) |
| query_params = urllib.parse.parse_qs(parsed_url.query) |
| for param_name, param_values in query_params.items(): |
| for value in param_values: |
| if value.startswith(('http://', 'https://')): |
| decoded_value = urllib.parse.unquote(value) |
| if any(ext in decoded_value.lower() for ext in constants.VIDEO_EXTENSIONS): |
| return decoded_value |
| decoded_url = urllib.parse.unquote(url) |
| if decoded_url != url and any(ext in decoded_url.lower() for ext in constants.VIDEO_EXTENSIONS): |
| return decoded_url |
| return url |
|
|
| |
| def get_domain_credibility(url): |
| domain = urllib.parse.urlparse(url).netloc.lower() |
| if domain.endswith(('.gov', '.edu', '.mil', 'arxiv.org', 'nature.com', 'science.org', 'nasa.gov')): |
| return "High_Scientific_Gov" |
| elif domain.endswith(('.org', '.com', '.net')) and any(x in domain for x in ['news', 'bbc', 'cnn', 'reuters', 'natgeo', 'space.com']): |
| return "Medium_Mainstream_Media" |
| elif any(x in domain for x in ['reddit', 'medium', 'quora', 'blogspot', 'wordpress', 'pinterest']): |
| return "Low_Fringe_UGC" |
| else: |
| return "Unknown_Evaluate" |
|
|
| |
| def is_banned_domain(url: str) -> bool: |
| try: |
| domain = urllib.parse.urlparse(url).netloc.lower() |
| |
| banned_keywords = ['tiktok.com', 'pinterest.', 'dailymotion.com', 'instagram.com', 'facebook.com', 'wattpad.com'] |
| return any(banned in domain for banned in banned_keywords) |
| except: |
| return False |
|
|
| |
| def get_youtube_video_id(url: str): |
| try: |
| parsed_url = urllib.parse.urlparse(url) |
| if 'youtube.com' in parsed_url.netloc: |
| if parsed_url.path == '/watch': |
| return urllib.parse.parse_qs(parsed_url.query).get('v', [None])[0] |
| elif parsed_url.path.startswith(('/embed/', '/v/')): |
| return parsed_url.path.split('/')[2] |
| elif 'youtu.be' in parsed_url.netloc: |
| return parsed_url.path[1:] |
| except: |
| pass |
| return None |
|
|
