File size: 9,640 Bytes
e0b68d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 | #!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
用户代理工具模块
提供动态随机用户代理生成功能，包含更丰富的浏览器指纹特征
"""
import random
import re
from typing import Any, Dict, Optional, Tuple
try:
from fake_useragent import UserAgent as _FakeUserAgent
except ImportError:
_FakeUserAgent = None
# 全局 UserAgent 实例（单例模式）
_user_agent_instance: Optional[Any] = None
def get_user_agent_instance() -> Any:
"""获取或创建 UserAgent 实例（单例模式）"""
global _user_agent_instance
if _user_agent_instance is None:
if _FakeUserAgent is None:
raise RuntimeError("fake_useragent is not installed")
try:
# 开启缓存以提高性能
_user_agent_instance = _FakeUserAgent(use_external_data=True, cache=True)
except Exception:
# 如果外部数据加载失败，提供降级机制
_user_agent_instance = _FakeUserAgent(use_external_data=False)
return _user_agent_instance
def get_random_user_agent(browser_type: Optional[str] = None) -> str:
"""
获取随机用户代理字符串
Args:
browser_type: 指定浏览器类型 ('chrome', 'firefox', 'safari', 'edge', 'mobile_safari', 'mobile_chrome')
如果为 None，则随机选择
Returns:
str: 用户代理字符串
"""
try:
ua = get_user_agent_instance()
# 定义更多的浏览器选择及其权重
if browser_type is None:
browser_choices = [
"chrome", "chrome", "chrome", "chrome",
"edge", "edge",
"firefox", "safari",
"mobile_chrome", "mobile_safari"
]
browser_type = random.choice(browser_choices)
# 根据浏览器类型获取用户代理
if browser_type == "chrome":
user_agent = ua.chrome
elif browser_type == "edge":
user_agent = ua.edge
elif browser_type == "firefox":
user_agent = ua.firefox
elif browser_type == "safari":
user_agent = ua.safari
# 对于移动端特定的 UA, fake_useragent.random 支持通过 OS 等获取，这里为增加可靠性可以显式定义些移动端特征
elif browser_type == "mobile_chrome":
# 尝试生成包含 Android 的 Chrome UA
user_agent = f"Mozilla/5.0 (Linux; Android {random.randint(10, 14)}; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{random.randint(110, 125)}.0.0.0 Mobile Safari/537.36"
elif browser_type == "mobile_safari":
ios_ver = f"{random.randint(15, 17)}_{random.randint(0, 5)}"
user_agent = f"Mozilla/5.0 (iPhone; CPU iPhone OS {ios_ver} like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/{random.randint(15, 17)}.0 Mobile/15E148 Safari/604.1"
else:
user_agent = ua.random
# 确保不返回 None 或空字符串
if not user_agent:
raise ValueError("获取到的 User-Agent 为空")
return user_agent
except Exception:
# Fallback 机制：如果 fake_useragent 失败，使用硬编码的常用 UA 列表
fallback_uas = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15"
]
return random.choice(fallback_uas)
def parse_ua_features(user_agent: str) -> Tuple[str, str, str, str, str]:
"""
解析 UA 字符串以提取平台、移动端标识、Chrome/Edge/Firefox 等版本号
"""
platform = '"Windows"'
if "Windows" in user_agent:
platform = '"Windows"'
elif "Macintosh" in user_agent or "Mac OS X" in user_agent:
platform = '"macOS"'
elif "Android" in user_agent:
platform = '"Android"'
elif "Linux" in user_agent:
platform = '"Linux"'
elif "iPhone" in user_agent or "iPad" in user_agent:
platform = '"iOS"'
elif "CrOS" in user_agent:
platform = '"Chrome OS"'
is_mobile = "?1" if "Mobile" in user_agent or "Android" in user_agent or platform in ('"Android"', '"iOS"') else "?0"
chrome_version = "124"
edge_version = "124"
firefox_version = ""
# 解析 Chrome 版本
chrome_match = re.search(r"Chrome/(\d+)\.", user_agent)
if chrome_match:
chrome_version = chrome_match.group(1)
# 解析 Edge 版本 (Edge, Edg, EdgiOS, EdgA)
edge_match = re.search(r"Edg[a-zA-Z]*/(\d+)\.", user_agent)
if edge_match:
edge_version = edge_match.group(1)
# 解析 Firefox 版本
firefox_match = re.search(r"Firefox/(\d+)\.", user_agent)
if firefox_match:
firefox_version = firefox_match.group(1)
return platform, is_mobile, chrome_version, edge_version, firefox_version
def get_dynamic_headers(
referer: Optional[str] = None,
origin: Optional[str] = None,
browser_type: Optional[str] = None,
additional_headers: Optional[Dict[str, str]] = None
) -> Dict[str, str]:
"""
生成动态浏览器 headers，包含随机 User-Agent 以及逼真的客户端提示 (Client Hints)
Args:
referer: 引用页面 URL
origin: 源站 URL
browser_type: 指定浏览器类型
additional_headers: 额外的 headers
Returns:
Dict[str, str]: 包含动态 User-Agent 的 headers
"""
user_agent = get_random_user_agent(browser_type)
# 随机选择常见的支持语言组合
accept_languages = [
"zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
"zh-CN,zh;q=0.9,en;q=0.8",
"zh-CN,zh-TW;q=0.9,zh;q=0.8,en-US;q=0.7,en;q=0.6",
"zh-CN,zh;q=0.9",
"en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
]
# 基础 headers
headers = {
"User-Agent": user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Accept-Language": random.choice(accept_languages),
"Accept-Encoding": "gzip, deflate, br, zstd",
"Cache-Control": "max-age=0",
"Connection": "keep-alive",
"DNT": "1" if random.random() > 0.5 else "0", # 随机的 Do Not Track
"Upgrade-Insecure-Requests": "1",
}
# 添加可选的 headers
if referer:
headers["Referer"] = referer
if origin:
headers["Origin"] = origin
# 解析特征
platform, is_mobile, chrome_version, edge_version, firefox_version = parse_ua_features(user_agent)
# 识别是否是 Chromium 内核浏览器 (包含 Chrome, Edge 等)
is_chromium = "Chrome/" in user_agent or "CrOS" in user_agent
is_edge = re.search(r"Edg[a-zA-Z]*/\d+\.", user_agent) is not None
is_safari = "Safari/" in user_agent and "Chrome/" not in user_agent and "Chromium/" not in user_agent
is_firefox = "Firefox/" in user_agent
# 针对 Chromium 内核（Chrome, Edge, Opera 等）添加特有的 Sec-CH-UA 头部
if is_chromium:
# Chromium 随机品牌策略
brand_list = [
f'"Not-A.Brand";v="99"',
f'"Not/A)Brand";v="8"',
f'"Chromium";v="{chrome_version}"'
]
if is_edge:
brand_list.append(f'"Microsoft Edge";v="{edge_version}"')
else:
brand_list.append(f'"Google Chrome";v="{chrome_version}"')
# 打乱品牌顺序使其更随机
random.shuffle(brand_list)
# 通常 Sec-CH-UA 有 3 个品牌
sec_ch_ua = ", ".join(brand_list[:3])
headers.update({
"sec-ch-ua": sec_ch_ua,
"sec-ch-ua-mobile": is_mobile,
"sec-ch-ua-platform": platform,
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "cross-site" if origin else "same-origin",
"Sec-Fetch-User": "?1",
})
elif is_firefox:
# Firefox 特有请求头
headers.update({
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "cross-site" if origin else "same-origin",
"Sec-Fetch-User": "?1",
"TE": "trailers",
})
elif is_safari:
# Safari 特有请求头
headers.update({
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "cross-site" if origin else "same-origin",
})
# 添加现代浏览器常见的优先度请求头
if "Priority" not in headers and (is_chromium or is_firefox):
headers["Priority"] = "u=0, i"
# 添加额外的 headers (覆盖默认值)
if additional_headers:
headers.update(additional_headers)
return headers
|