import axios from 'axios'; // 根据你的目录结构,向上跳两级到 src,再进入 config import { getGlobalConfig } from "../../config/config"; const JINA_API_KEY = getGlobalConfig() || process.env.JINA_API_KEY; const READER_ENDPOINT = "https://r.jina.ai/"; export interface FetchResult { url: string; finalUrl: string; status: number; extractor: string; truncated: boolean; length: number; untrusted: boolean; text: string; } /** * 延迟函数 */ function delay(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } /** * 带重试的请求函数 */ async function fetchWithRetry( requestFn: () => Promise, maxRetries: number = 3, retryDelay: number = 1000 ): Promise { let lastError: any; for (let attempt = 1; attempt <= maxRetries; attempt++) { try { return await requestFn(); } catch (error: any) { lastError = error; // 如果是服务器返回的错误(4xx, 5xx),不重试 if (error.response) { throw error; } // 如果是网络错误或超时,进行重试 if (attempt < maxRetries) { console.warn(`Jina Fetch: 网络错误,第 ${attempt} 次尝试失败,${retryDelay}ms 后重试...`); await delay(retryDelay); } } } throw lastError; } /** * 使用 Jina Reader API 抓取网页内容 * @param url 目标网页 URL * @param maxChars 最大字符截断限制 */ export async function jinaFetch(url: string, maxChars: number = 50000): Promise { // 1. 基础校验:如果 URL 为空或明显非法,直接返回 null if (!url || typeof url !== 'string' || url.trim().length === 0) { console.warn("Jina Fetch: Invalid or empty URL provided."); return null; } try { // 2. 安全拼接 URL // 使用 encodeURIComponent 确保目标 URL 中的特殊字符不会破坏请求格式 const requestUrl = `${READER_ENDPOINT}${encodeURIComponent(url)}`; // 3. 使用重试机制发送请求 const response = await fetchWithRetry(async () => { return await axios.get(requestUrl, { headers: { // 只有当 API_KEY 存在时才添加 Authorization ...(JINA_API_KEY ? { 'Authorization': `Bearer ${JINA_API_KEY}` } : {}), 'Accept': 'application/json', 'X-Return-Format': 'markdown' }, timeout: 30000 // 增加超时时间到 30 秒 }); }, 3, 1000); // 最多重试 3 次,每次间隔 1 秒 // 4. 处理频率限制 if (response.status === 429) { console.debug("Jina Reader rate limited"); return null; } // 5. 解析数据结构 // Jina API 返回结构: { code, status, data: { title, content, url, ... } } const apiResponse = response.data || {}; const data = apiResponse.data || {}; const title = data.title || ""; let content = data.content || ""; // 调试日志:记录响应结构 console.debug(`Jina API Response Status: ${response.status}`); console.debug(`Response data keys: ${Object.keys(data).join(', ')}`); console.debug(`Title length: ${title.length}, Content length: ${content.length}`); if (!content) { console.warn(`Jina Fetch: No content found in response for URL: ${url}`); console.warn(`Response data:`, JSON.stringify(data, null, 2)); return null; } // 6. 格式化内容 let fullText = title ? `# ${title}\n\n${content}` : content; const isTruncated = fullText.length > maxChars; if (isTruncated) { fullText = fullText.slice(0, maxChars); } const result: FetchResult = { url: url, finalUrl: data.url || url, status: response.status, extractor: "jina", truncated: isTruncated, length: fullText.length, untrusted: true, text: fullText }; return JSON.stringify(result, null, 2); } catch (error: any) { // 7. 增强错误日志输出 if (error.response) { // 服务器响应了错误(如 400, 403, 404) const status = error.response.status; const errorDetail = JSON.stringify(error.response.data); console.error(`Jina API Error (Status ${status}): ${errorDetail}`); } else if (error.request) { // 请求已发出但未收到响应 console.error(`Jina Fetch No Response: ${error.message}`); } else { // 设置请求时发生错误 console.error(`Jina Fetch Configuration Error: ${error.message}`); } return null; } }