src/config.ts

/**
 * 爬虫配置文件
 * 集中管理所有爬虫配置,方便切换和调整
 */

export interface CrawlerConfig {
  preset: string;
  maxConcurrency: number;
  requestHandlerTimeoutSecs: number;
  maxRequestRetries: number;
  pageLoadWaitMs: number;
  launchOptions: {
    headless: boolean | 'new';
    args?: string[];
  };
  selectors: {
    brand: string[];
    price: string[];
    originalPrice: string[];
    rating: string[];
    reviews: string[];
  };
  // 添加代理配置
  useProxy: boolean;
  proxyFile?: string;
}

const PRESETS: Record<string, Partial<CrawlerConfig>> = {
  fast: {
    maxConcurrency: 20,
    requestHandlerTimeoutSecs: 30,
    maxRequestRetries: 1,
    pageLoadWaitMs: 500,
    useProxy: true, // 启用代理
    launchOptions: { 
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-gpu',
        '--disable-software-rasterizer',
        '--disable-extensions',
        '--disable-background-networking',
        '--disable-default-apps',
        '--disable-sync',
        '--disable-translate',
        '--hide-scrollbars',
        '--mute-audio',
        '--no-first-run',
        '--no-pings',
        '--disable-features=IsolateOrigins,site-per-process'
      ]
    }
  },
  stable: {
    maxConcurrency: 5,
    requestHandlerTimeoutSecs: 90,
    maxRequestRetries: 1,
    pageLoadWaitMs: 500,
    useProxy: true, // 启用代理
    launchOptions: { 
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-gpu',
        '--disable-software-rasterizer',
        '--disable-extensions',
        '--disable-background-networking', // 禁止后台网络活动
        '--disable-default-apps',
        '--disable-sync',
        '--disable-translate',
        '--hide-scrollbars',
        '--mute-audio',
        '--no-first-run',
        '--no-pings',
        '--disable-features=IsolateOrigins,site-per-process',
        '--dns-prefetch-disable', // 禁用 DNS 预取，有时能避免奇怪的阻塞
        '--disable-logging'
      ]
    }
  },
  stealth: {
    maxConcurrency: 2, // 进一步降低并发数
    requestHandlerTimeoutSecs: 240, // 延长超时时间
    maxRequestRetries: 2, // 增加重试次数
    pageLoadWaitMs: 800, // 增加页面等待时间
    useProxy: false, // 启用代理
    launchOptions: { 
      headless: 'new',
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-gpu',
        '--disable-software-rasterizer',
        '--disable-extensions',
        '--disable-background-networking',
        '--disable-default-apps',
        '--disable-sync',
        '--disable-translate',
        '--hide-scrollbars',
        '--mute-audio',
        '--no-first-run',
        '--no-pings',
        '--disable-features=IsolateOrigins,site-per-process',
        // 新增反检测参数
        '--disable-blink-features=AutomationControlled',
        '--disable-automation',
        '--disable-web-security',
        '--allow-running-insecure-content',
        '--disable-features=VizDisplayCompositor',
        '--disable-features=ImprovedCookieControls,SameSiteByDefaultCookies,CookiesWithoutSameSiteMustBeSecure',
        '--disable-features=WebRtcHideLocalIpsWithMdns',
        '--disable-features=WebContentsForceDark',
        '--disable-features=AutofillEnableAccountWalletStorage',
        '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
      ]
    }
  }
};

export const DEFAULT_SELECTORS = {
  // 标题选择器 - 用于智能等待判断页面是否加载完成
  brand: [
    '.js_pdp_short-info__brand-link'
  ],
  price: [
    'span.js_pdp_price__retail-price__value_original'
  ],
  originalPrice: [
    'span.pdp_price__strike-through-price'
  ],
  rating: [
    'div.pdp_cr-rating-score span'
  ],
  reviews: [
    'span.js_pdp_cr-rating--review-count'
  ]
};

export function getCrawlerConfig(presetName: string = 'stable', overrides: Partial<CrawlerConfig> = {}): CrawlerConfig {
  const preset = PRESETS[presetName] || PRESETS['stable'];
  
  // 先合并基础配置
  const baseConfig: CrawlerConfig = {
    preset: presetName,
    maxConcurrency: 5,
    requestHandlerTimeoutSecs: 60,
    maxRequestRetries: 3,
    pageLoadWaitMs: 2000,
    useProxy: false, // 默认不使用代理
    selectors: DEFAULT_SELECTORS,
    launchOptions: {
       headless: 'new'
    }
  };
  
  // 合并预设配置
  const mergedConfig: Partial<CrawlerConfig> = {
    ...baseConfig,
    ...preset,
    ...overrides
  };
  
  // 特殊处理 launchOptions 的合并
  mergedConfig.launchOptions = {
    ...(baseConfig.launchOptions),
    ...(preset.launchOptions || {}),
    ...(overrides.launchOptions || {})
  };
  
  return mergedConfig as CrawlerConfig;
}