#![allow(dead_code)] use reqwest::blocking::Client; use reqwest::header::{ ACCEPT, ACCEPT_ENCODING, CONNECTION, CONTENT_TYPE, HeaderMap, HeaderValue, REFERER, USER_AGENT, }; use serde_json::Value; use std::fs; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tracing::{debug, error, warn}; // 编译一次复用的正则缓存 fn re_next_data() -> &'static regex::Regex { static R: OnceLock = OnceLock::new(); R.get_or_init(|| { regex::Regex::new(r#"(?s)]*id="__NEXT_DATA__"[^>]*>(.*?)"#).unwrap() }) } fn re_initial_state() -> &'static regex::Regex { static R: OnceLock = OnceLock::new(); R.get_or_init(|| { regex::Regex::new(r#"(?s)window\.__INITIAL_STATE__\s*=\s*(\{.*?\})\s*;"#).unwrap() }) } fn re_info_label_grey() -> &'static regex::Regex { static R: OnceLock = OnceLock::new(); R.get_or_init(|| { regex::Regex::new(r#"]*class="info-label-grey"[^>]*>([^<]+)"#).unwrap() }) } fn re_info_label_yellow() -> &'static regex::Regex { static R: OnceLock = OnceLock::new(); R.get_or_init(|| { regex::Regex::new(r#"]*class="info-label-yellow"[^>]*>([^<]+)"#).unwrap() }) } /// 匹配 HTML `` 中 `` 块。 /// 封面图的真实 URL（带签名参数）位于其中的 `"image"` 或 `"images"` 字段。 fn re_ld_json() -> &'static regex::Regex { static R: OnceLock = OnceLock::new(); R.get_or_init(|| { regex::Regex::new( r#"]*type="application/ld\+json"[^>]*>\s*([\s\S]*?)\s*"#, ) .unwrap() }) } #[derive(Debug, Clone, Default)] pub(crate) struct BookInfo { pub book_name: Option, pub author: Option, pub description: Option, pub tags: Option>, pub cover_url: Option, pub detail_cover_url: Option, /// HTML `` 的 src，浏览器兼容格式（非 HEIC）。 pub html_img_cover_url: Option, pub chapter_count: Option, pub finished: Option, } #[derive(Debug, Clone)] pub(crate) struct FanqieWebConfig { pub request_timeout: Duration, pub max_retries: usize, pub insecure_tls: bool, pub user_agent: String, pub cache_dir: PathBuf, } impl Default for FanqieWebConfig { fn default() -> Self { Self { request_timeout: Duration::from_secs(15), max_retries: 3, insecure_tls: false, user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36".to_string(), cache_dir: std::env::temp_dir().join("tomato-novel-downloader").join("dir_cache"), } } } pub(crate) struct FanqieWebNetwork { client: Client, config: FanqieWebConfig, last_dir_fetch: Mutex, } pub(crate) type BookInfoParts = ( Option, Option, Option, Option>, Option, Option, Option, // html_img_cover_url Option, Option, ); impl FanqieWebNetwork { pub(crate) fn new(config: FanqieWebConfig) -> anyhow::Result { let mut default_headers = HeaderMap::new(); default_headers.insert(ACCEPT_ENCODING, HeaderValue::from_static("identity")); default_headers.insert(CONNECTION, HeaderValue::from_static("keep-alive")); let client = Client::builder() .default_headers(default_headers) .danger_accept_invalid_certs(config.insecure_tls) .timeout(config.request_timeout) .build()?; Ok(Self { client, config, last_dir_fetch: Mutex::new(Instant::now() - Duration::from_secs(60)), }) } fn get_headers(&self) -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert( ACCEPT, HeaderValue::from_static( "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", ), ); headers.insert( USER_AGENT, HeaderValue::from_str(&self.config.user_agent) .unwrap_or(HeaderValue::from_static("Mozilla/5.0")), ); headers } fn get_json_headers(&self, book_id: &str) -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert( ACCEPT, HeaderValue::from_static("application/json, text/plain, */*"), ); headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); headers.insert( USER_AGENT, HeaderValue::from_str(&self.config.user_agent) .unwrap_or(HeaderValue::from_static("Mozilla/5.0")), ); let referer = format!("https://fanqienovel.com/page/{book_id}"); if let Ok(v) = HeaderValue::from_str(&referer) { headers.insert(REFERER, v); } headers } pub(crate) fn get_book_info(&self, book_id: &str) -> BookInfoParts { let book_info_url = format!("https://fanqienovel.com/page/{book_id}"); // 发送请求 match self .client .get(&book_info_url) .headers(self.get_headers()) .send() { Ok(resp) => { if resp.status().as_u16() == 404 { error!("小说ID {} 不存在！", book_id); return (None, None, None, None, None, None, None, None, None); } let resp = match resp.error_for_status() { Ok(r) => r, Err(e) => { error!("获取书籍信息失败: {}", e); return (None, None, None, None, None, None, None, None, None); } }; match resp.text() { Ok(text) => { let info = ContentParser::parse_book_info(&text, book_id); ( info.book_name, info.author, info.description, info.tags, info.cover_url, info.detail_cover_url, info.html_img_cover_url, info.chapter_count, info.finished, ) } Err(e) => { error!("获取书籍信息失败: {}", e); (None, None, None, None, None, None, None, None, None) } } } Err(e) => { error!("获取书籍信息失败: {}", e); (None, None, None, None, None, None, None, None, None) } } } /// 从 web API 获取章节列表（节流 + 403 预热 + 退避重试 + 本地缓存回退）。 pub(crate) fn fetch_chapter_list(&self, book_id: &str) -> Option> { // 无效 book_id 直接返回 None，避免无意义请求 if book_id.trim().is_empty() || !book_id.chars().all(|c| c.is_ascii_digit()) { warn!("fetch_chapter_list 跳过无效 book_id: '{}'", book_id); return None; } let api_url = format!("https://fanqienovel.com/api/reader/directory/detail?bookId={book_id}"); // 节流：与上次请求间隔至少 0.8s，降低被限频概率 self.throttle_directory(Duration::from_millis(800)); let retries = self.config.max_retries.max(1); let mut backoff = 0.6f64; let mut last_error: Option = None; for attempt in 1..=retries { debug!("开始获取章节列表，URL: {}", api_url); let headers = self.get_json_headers(book_id); if attempt == 1 { // 屏蔽 Cookie（如果未来启用 cookies feature，这里也不会泄露） let masked: Vec<(String, String)> = headers .iter() .map(|(k, v)| { let key = k.as_str().to_string(); let val = if key.eq_ignore_ascii_case("cookie") { "***".to_string() } else { v.to_str().unwrap_or("").to_string() }; (key, val) }) .collect(); debug!("目录请求Header(精简): {:?}", masked); } else { debug!( "重试第 {} 次获取目录（可能被限频/风控），URL: {}", attempt, api_url ); } let resp = self.client.get(&api_url).headers(headers).send(); let resp = match resp { Ok(r) => r, Err(e) => { last_error = Some(e.to_string()); error!("获取章节列表失败: {}", e); self.sleep_backoff(attempt, retries, &mut backoff, 0.3); continue; } }; debug!("章节列表响应状态: {}", resp.status().as_u16()); // 显式处理 403：可能为风控或限频 if resp.status().as_u16() == 403 { last_error = Some("403 Forbidden".to_string()); // 首次遇到 403 时，尝试预热页面以获取必要 Cookie，再退避重试 if attempt == 1 { let warm_url = format!("https://fanqienovel.com/page/{book_id}"); match self .client .get(&warm_url) .headers(self.get_headers()) .send() { Ok(_) => { debug!("已尝试通过页面预热获取 Cookie，准备退避后重试目录 API"); } Err(e) => { debug!("页面预热失败: {}", e); } } } self.sleep_backoff(attempt, retries, &mut backoff, 0.4); continue; } let resp = match resp.error_for_status() { Ok(r) => r, Err(e) => { last_error = Some(e.to_string()); error!("获取章节列表失败: {}", e); self.sleep_backoff(attempt, retries, &mut backoff, 0.3); continue; } }; let data: Value = match resp.json() { Ok(v) => v, Err(e) => { last_error = Some(e.to_string()); error!("获取章节列表失败: {}", e); self.sleep_backoff(attempt, retries, &mut backoff, 0.3); continue; } }; // 成功则缓存原始 JSON，便于下次回退 if let Err(e) = self.save_dir_cache(book_id, &data) { debug!("保存目录缓存失败(忽略): {}", e); } if let Some(list) = Self::parse_chapter_data(&data) { return Some(list); } last_error = Some("parse chapter list failed".to_string()); warn!("获取章节列表失败: 解析章节数组为空"); self.sleep_backoff(attempt, retries, &mut backoff, 0.3); continue; } debug!("重试仍失败：{:?}", last_error); // 重试仍失败：尝试使用本地缓存回退 match self.load_dir_cache(book_id) { Ok(Some(cached)) => { debug!("使用本地缓存的章节目录回退: book_id={}", book_id); Self::parse_chapter_data(&cached) } _ => None, } } fn throttle_directory(&self, min_gap: Duration) { if let Ok(mut last) = self.last_dir_fetch.lock() { let elapsed = last.elapsed(); if elapsed < min_gap { std::thread::sleep(min_gap - elapsed); } *last = Instant::now(); } } fn sleep_backoff(&self, attempt: usize, retries: usize, backoff: &mut f64, jitter_max: f64) { if attempt >= retries { return; } let jitter = jitter_seconds(jitter_max); let sleep_s = (*backoff + jitter).min(3.0); std::thread::sleep(Duration::from_millis((sleep_s * 1000.0) as u64)); *backoff = (*backoff * 2.0).min(3.0); } fn cache_path(&self, book_id: &str) -> PathBuf { self.config.cache_dir.join(format!("{book_id}.json")) } fn save_dir_cache(&self, book_id: &str, data: &Value) -> anyhow::Result<()> { let path = self.cache_path(book_id); if let Some(parent) = path.parent() { fs::create_dir_all(parent)?; } let bytes = serde_json::to_vec(data)?; fs::write(path, bytes)?; Ok(()) } fn load_dir_cache(&self, book_id: &str) -> anyhow::Result> { let path = self.cache_path(book_id); if !path.exists() { return Ok(None); } let bytes = fs::read(path)?; let value: Value = serde_json::from_slice(&bytes)?; Ok(Some(value)) } fn parse_chapter_data(data: &Value) -> Option> { // 兼容多种返回形态：尽量提取出“章节数组”。 let root = data.get("data").unwrap_or(data); for key in [ "chapterList", "chapter_list", "chapters", "item_list", "items", "list", ] { if let Some(arr) = root.get(key).and_then(Value::as_array) { return Some(arr.clone()); } } // chapterListWithVolume：按卷分组的章节列表（数组的数组），需要展平 if let Some(volumes) = root.get("chapterListWithVolume").and_then(Value::as_array) { let mut all_chapters: Vec = Vec::new(); for vol in volumes { if let Some(ch_list) = vol.as_array() { all_chapters.extend(ch_list.iter().cloned()); } } if !all_chapters.is_empty() { return Some(all_chapters); } } // 有些接口会是 data.data.list / data.data.chapterList / data.data.items if let Some(inner) = root.get("data") { for key in [ "list", "chapterList", "chapter_list", "items", "item_list", "chapters", ] { if let Some(arr) = inner.get(key).and_then(Value::as_array) { return Some(arr.clone()); } } } // 最后兜底：递归扫描 JSON，找到“像章节数组”的最大数组 find_chapter_array(root) } } fn is_chapter_like_object(map: &serde_json::Map) -> bool { let keys = [ "item_id", "itemId", "chapter_id", "chapterId", "catalog_id", "catalogId", "id", ]; keys.iter().any(|k| map.contains_key(*k)) } fn find_chapter_array(value: &Value) -> Option> { fn walk(value: &Value, best: &mut Option>) { match value { Value::Array(arr) => { let is_candidate = arr .iter() .any(|v| v.as_object().map(is_chapter_like_object).unwrap_or(false)); if is_candidate { let replace = match best { Some(existing) => arr.len() > existing.len(), None => true, }; if replace { *best = Some(arr.clone()); } } for v in arr { walk(v, best); } } Value::Object(map) => { for v in map.values() { walk(v, best); } } _ => {} } } let mut best: Option> = None; walk(value, &mut best); best } /// 从 HTML `` 中的 ` "#; let info = ContentParser::parse_book_info(html, "dummy"); assert_eq!(info.finished, Some(true)); } #[test] fn finished_status_mapping_for_status_field() { let html_finished = r#" "#; let info_finished = ContentParser::parse_book_info(html_finished, "dummy"); assert_eq!(info_finished.finished, Some(true)); let html_serializing = r#" "#; let info_serializing = ContentParser::parse_book_info(html_serializing, "dummy"); assert_eq!(info_serializing.finished, Some(false)); } }