File size: 4,527 Bytes
1bf9e0c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 | import axios from 'axios';
// 根据你的目录结构，向上跳两级到 src，再进入 config
import { getGlobalConfig } from "../../config/config";
const JINA_API_KEY = getGlobalConfig() || process.env.JINA_API_KEY;
const READER_ENDPOINT = "https://r.jina.ai/";
export interface FetchResult {
url: string;
finalUrl: string;
status: number;
extractor: string;
truncated: boolean;
length: number;
untrusted: boolean;
text: string;
}
/**
* 延迟函数
*/
function delay(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* 带重试的请求函数
*/
async function fetchWithRetry(
requestFn: () => Promise<any>,
maxRetries: number = 3,
retryDelay: number = 1000
): Promise<any> {
let lastError: any;
for (let attempt = 1; attempt <= maxRetries; attempt++) {
try {
return await requestFn();
} catch (error: any) {
lastError = error;
// 如果是服务器返回的错误（4xx, 5xx），不重试
if (error.response) {
throw error;
}
// 如果是网络错误或超时，进行重试
if (attempt < maxRetries) {
console.warn(`Jina Fetch: 网络错误，第 ${attempt} 次尝试失败，${retryDelay}ms 后重试...`);
await delay(retryDelay);
}
}
}
throw lastError;
}
/**
* 使用 Jina Reader API 抓取网页内容
* @param url 目标网页 URL
* @param maxChars 最大字符截断限制
*/
export async function jinaFetch(url: string, maxChars: number = 50000): Promise<string | null> {
// 1. 基础校验：如果 URL 为空或明显非法，直接返回 null
if (!url || typeof url !== 'string' || url.trim().length === 0) {
console.warn("Jina Fetch: Invalid or empty URL provided.");
return null;
}
try {
// 2. 安全拼接 URL
// 使用 encodeURIComponent 确保目标 URL 中的特殊字符不会破坏请求格式
const requestUrl = `${READER_ENDPOINT}${encodeURIComponent(url)}`;
// 3. 使用重试机制发送请求
const response = await fetchWithRetry(async () => {
return await axios.get(requestUrl, {
headers: {
// 只有当 API_KEY 存在时才添加 Authorization
...(JINA_API_KEY ? { 'Authorization': `Bearer ${JINA_API_KEY}` } : {}),
'Accept': 'application/json',
'X-Return-Format': 'markdown'
},
timeout: 30000 // 增加超时时间到 30 秒
});
}, 3, 1000); // 最多重试 3 次，每次间隔 1 秒
// 4. 处理频率限制
if (response.status === 429) {
console.debug("Jina Reader rate limited");
return null;
}
// 5. 解析数据结构
// Jina API 返回结构: { code, status, data: { title, content, url, ... } }
const apiResponse = response.data || {};
const data = apiResponse.data || {};
const title = data.title || "";
let content = data.content || "";
// 调试日志：记录响应结构
console.debug(`Jina API Response Status: ${response.status}`);
console.debug(`Response data keys: ${Object.keys(data).join(', ')}`);
console.debug(`Title length: ${title.length}, Content length: ${content.length}`);
if (!content) {
console.warn(`Jina Fetch: No content found in response for URL: ${url}`);
console.warn(`Response data:`, JSON.stringify(data, null, 2));
return null;
}
// 6. 格式化内容
let fullText = title ? `# ${title}\n\n${content}` : content;
const isTruncated = fullText.length > maxChars;
if (isTruncated) {
fullText = fullText.slice(0, maxChars);
}
const result: FetchResult = {
url: url,
finalUrl: data.url || url,
status: response.status,
extractor: "jina",
truncated: isTruncated,
length: fullText.length,
untrusted: true,
text: fullText
};
return JSON.stringify(result, null, 2);
} catch (error: any) {
// 7. 增强错误日志输出
if (error.response) {
// 服务器响应了错误（如 400, 403, 404）
const status = error.response.status;
const errorDetail = JSON.stringify(error.response.data);
console.error(`Jina API Error (Status ${status}): ${errorDetail}`);
} else if (error.request) {
// 请求已发出但未收到响应
console.error(`Jina Fetch No Response: ${error.message}`);
} else {
// 设置请求时发生错误
console.error(`Jina Fetch Configuration Error: ${error.message}`);
}
return null;
}
}
|