src/tools/WebFetchTool/jina_fetch.ts

import axios from 'axios';
// 根据你的目录结构，向上跳两级到 src，再进入 config
import { getGlobalConfig } from "../../config/config";

const JINA_API_KEY = getGlobalConfig() || process.env.JINA_API_KEY;
const READER_ENDPOINT = "https://r.jina.ai/";

export interface FetchResult {
  url: string;
  finalUrl: string;
  status: number;
  extractor: string;
  truncated: boolean;
  length: number;
  untrusted: boolean;
  text: string;
}

/**
 * 延迟函数
 */
function delay(ms: number): Promise<void> {
  return new Promise(resolve => setTimeout(resolve, ms));
}

/**
 * 带重试的请求函数
 */
async function fetchWithRetry(
  requestFn: () => Promise<any>,
  maxRetries: number = 3,
  retryDelay: number = 1000
): Promise<any> {
  let lastError: any;

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await requestFn();
    } catch (error: any) {
      lastError = error;

      // 如果是服务器返回的错误（4xx, 5xx），不重试
      if (error.response) {
        throw error;
      }

      // 如果是网络错误或超时，进行重试
      if (attempt < maxRetries) {
        console.warn(`Jina Fetch: 网络错误，第 ${attempt} 次尝试失败，${retryDelay}ms 后重试...`);
        await delay(retryDelay);
      }
    }
  }

  throw lastError;
}

/**
 * 使用 Jina Reader API 抓取网页内容
 * @param url 目标网页 URL
 * @param maxChars 最大字符截断限制
 */
export async function jinaFetch(url: string, maxChars: number = 50000): Promise<string | null> {
  // 1. 基础校验：如果 URL 为空或明显非法，直接返回 null
  if (!url || typeof url !== 'string' || url.trim().length === 0) {
    console.warn("Jina Fetch: Invalid or empty URL provided.");
    return null;
  }

  try {
    // 2. 安全拼接 URL
    // 使用 encodeURIComponent 确保目标 URL 中的特殊字符不会破坏请求格式
    const requestUrl = `${READER_ENDPOINT}${encodeURIComponent(url)}`;

    // 3. 使用重试机制发送请求
    const response = await fetchWithRetry(async () => {
      return await axios.get(requestUrl, {
        headers: {
          // 只有当 API_KEY 存在时才添加 Authorization
          ...(JINA_API_KEY ? { 'Authorization': `Bearer ${JINA_API_KEY}` } : {}),
          'Accept': 'application/json',
          'X-Return-Format': 'markdown'
        },
        timeout: 30000 // 增加超时时间到 30 秒
      });
    }, 3, 1000); // 最多重试 3 次，每次间隔 1 秒

    // 4. 处理频率限制
    if (response.status === 429) {
      console.debug("Jina Reader rate limited");
      return null;
    }

    // 5. 解析数据结构
    // Jina API 返回结构: { code, status, data: { title, content, url, ... } }
    const apiResponse = response.data || {};
    const data = apiResponse.data || {};
    const title = data.title || "";
    let content = data.content || "";

    // 调试日志：记录响应结构
    console.debug(`Jina API Response Status: ${response.status}`);
    console.debug(`Response data keys: ${Object.keys(data).join(', ')}`);
    console.debug(`Title length: ${title.length}, Content length: ${content.length}`);

    if (!content) {
      console.warn(`Jina Fetch: No content found in response for URL: ${url}`);
      console.warn(`Response data:`, JSON.stringify(data, null, 2));
      return null;
    }

    // 6. 格式化内容
    let fullText = title ? `# ${title}\n\n${content}` : content;

    const isTruncated = fullText.length > maxChars;
    if (isTruncated) {
      fullText = fullText.slice(0, maxChars);
    }

    const result: FetchResult = {
      url: url,
      finalUrl: data.url || url,
      status: response.status,
      extractor: "jina",
      truncated: isTruncated,
      length: fullText.length,
      untrusted: true,
      text: fullText
    };

    return JSON.stringify(result, null, 2);

  } catch (error: any) {
    // 7. 增强错误日志输出
    if (error.response) {
      // 服务器响应了错误（如 400, 403, 404）
      const status = error.response.status;
      const errorDetail = JSON.stringify(error.response.data);
      console.error(`Jina API Error (Status ${status}): ${errorDetail}`);
    } else if (error.request) {
      // 请求已发出但未收到响应
      console.error(`Jina Fetch No Response: ${error.message}`);
    } else {
      // 设置请求时发生错误
      console.error(`Jina Fetch Configuration Error: ${error.message}`);
    }

    return null;
  }
}