src/otto-crawler.ts

import { PlaywrightCrawler, ProxyConfiguration, Dataset, log } from 'crawlee';
import { Page } from 'playwright';
import { getCrawlerConfig, CrawlerConfig } from './config.js';
import { ProductData } from './productData.js';
import { createProxyConfiguration } from './proxy-config.js';
// 导入反检测模块
import { injectAntiDetectionScripts, simulateHumanBehavior } from './anti-detection.js';

// 默认配置
let currentConfig: CrawlerConfig = getCrawlerConfig();

/**
 * 从页面提取文本内容
 */
async function extractText(page: Page, ...selectors: string[]): Promise<string | null> {
  for (const selector of selectors) {
    try {
      const locator = page.locator(selector).first();
      const count = await locator.count();
      
      if (count > 0) {
        const text = await locator.textContent();
        if (text && text.trim()) {
          return text.trim();
        }
      }
    } catch (error: any) {
      log.debug(`文本选择器失败:${selector}`, error.message);
    }
  }
  return null;
}

/**
 * 提取 Otto 商品详情数据
 */
async function extractOttoData(page: Page, product: ProductData = {} as ProductData): Promise<ProductData> {
  try {
    const selectors = currentConfig.selectors;

    // 提取品牌
    const brand = await extractText(page, ...selectors.brand);
    if (brand) product.brand = brand;
    
    // 提取价格
    const price = await extractText(page, ...selectors.price);
    if (price) product.price = price;

    const originalPrice = await extractText(page, ...selectors.originalPrice);
    if (price) product.originalPrice = originalPrice;
    
    // 提取评分
    const rating = await extractText(page, ...selectors.rating);
    if (rating) product.rating = rating;
    
    // 提取评论数
    const reviews = await extractText(page, ...selectors.reviews);
    if (reviews) product.reviews = reviews;
    
    log.info(`✓ 成功提取商品数据:`, {
      brand: product.brand || 'N/A',
      price: product.price || 'N/A',
      rating: product.rating || 'N/A',
      reviews: product.reviews || 'N/A'
    });
    
    return product;
  } catch (error: any) {
    log.warning('Otto 数据提取失败:', error.message);
    return product;
  }
}

/**
 * 创建 Otto 商品爬虫实例
 */
function createOttoCrawler(options: { preset?: string } & Partial<CrawlerConfig> = {}) {
  // 合并配置
  const config = getCrawlerConfig(options.preset, options);
  currentConfig = config; // 更新当前配置
  
  // 处理 headless 类型转换
  const launchOptions: any = {
    ...config.launchOptions,
    headless: config.launchOptions.headless === 'new' ? true : config.launchOptions.headless,
    // 优化：增加浏览器级别的超时时间
    timeout: 120000, // 120秒（从默认30秒增加）11
  };
  
  // 获取代理配置
  const proxyConfiguration = createProxyConfiguration();
  
  const crawler = new PlaywrightCrawler({
    maxConcurrency: config.maxConcurrency,
    requestHandlerTimeoutSecs: config.requestHandlerTimeoutSecs,
    maxRequestRetries: config.maxRequestRetries,
    navigationTimeoutSecs: 120, // 导航超时120秒
    // 使用 Decodo 代理配置
    proxyConfiguration: proxyConfiguration,
    launchContext: {
      launchOptions
    },
    preNavigationHooks: [
      async ({ page }) => {
        // 注入反检测脚本
        await injectAntiDetectionScripts(page);
        
        // 拦截并 abort 非必要的资源类型
        await page.route('**/*', (route, request) => {
          const resourceType = request.resourceType();
          // 只拦截图片、字体、媒体文件,保留 stylesheet 以确保布局正确
          if (['image', 'font', 'media'].includes(resourceType)) {
            route.abort();
          } else {
            route.continue();
          }
        });
      }
    ],
    async requestHandler({ request, page, log }) {
      const startTime = Date.now();
      log.info(`🔄 开始处理: ${request.url}`);
      
      // 1. 等待 DOM 加载完成
      await page.waitForLoadState('domcontentloaded');
      
      // 2. 模拟真实用户行为（包含必要的等待）
      await simulateHumanBehavior(page);
      
      // 3. 等待关键元素出现
      const waitStartTime = Date.now();
      try {
        await page.waitForSelector('span.js_pdp_cr-rating--review-count', { 
          state: 'visible', 
          timeout: 5000 // 从3秒增加到5秒，给页面更多时间
        });
        log.info(`⏱️ 元素等待耗时: ${Date.now() - waitStartTime}ms`);
      } catch (e) {
        log.warning('⚠️ 未检测到关键元素,但将继续尝试提取数据');
      }
      
      // 4. 额外等待确保数据完全加载（平衡点）
      await page.waitForTimeout(300);
      
      const productData: ProductData = {
        url: request.url,
        crawledAt: new Date().toISOString()
      };
      
      const extractStartTime = Date.now();
      const extractedData = await extractOttoData(page, productData);
      const extractTime = Date.now() - extractStartTime;
      log.info(`⏱️ 数据提取耗时: ${extractTime}ms`);
      
      await Dataset.pushData(extractedData);
      
      const totalTime = Date.now() - startTime;
      log.info(`✅ 完成处理: ${request.url} (总耗时: ${totalTime}ms)`);
    },
    // 修复弃用警告: error 作为第二个参数传入
    async failedRequestHandler({ request, log }, error) {
      log.error(`请求失败 (重试 ${request.retryCount}/${config.maxRequestRetries}): ${request.url}`, { 
        message: (error as Error).message 
      });
    },
    // 修复弃用警告: error 作为第二个参数传入
    errorHandler({ log }, error) {
      log.error('爬虫全局错误:', { message: (error as Error).message });
    }
  });
  
  return crawler;
}

/**
 * 运行 Otto 商品爬虫
 */
async function runOttoCrawler(urls: string | string[], options: { preset?: string } & Partial<CrawlerConfig> = {}): Promise<ProductData[]> {
  const overallStart = Date.now();
  const urlList = Array.isArray(urls) ? urls : [urls];
  
  log.info('🚀 开始 Otto 商品爬虫...');
  log.info(`📋 待爬取 URL 数量: ${urlList.length}`);
  log.info(`🔧 配置预设: ${options.preset || 'default'}`);
  
  // Step 1: 创建爬虫实例
  const createStart = Date.now();
  const crawler = createOttoCrawler(options);
  const createDuration = Date.now() - createStart;
  log.info(`⏱️  [步骤1] 创建爬虫实例耗时: ${createDuration}ms`);
  
  // Step 2: 添加请求到队列
  const addRequestsStart = Date.now();
  await crawler.addRequests(urlList.map(url => ({
    url,
    userData: { label: 'DETAIL' }
  })));
  const addRequestsDuration = Date.now() - addRequestsStart;
  log.info(`⏱️  [步骤2] 添加请求到队列耗时: ${addRequestsDuration}ms`);
  
  // Step 3: 执行爬虫
  log.info('🔄 开始执行爬取任务...');
  const runStart = Date.now();
  await crawler.run();
  const runDuration = Date.now() - runStart;
  log.info(`⏱️  [步骤3] 执行爬取任务耗时: ${runDuration}ms (${(runDuration / 1000).toFixed(2)}s)`);
  
  // Step 4: 获取结果数据
  const getDataStart = Date.now();
  const results = await Dataset.getData();
  const getDataDuration = Date.now() - getDataStart;
  log.info(`⏱️  [步骤4] 获取结果数据耗时: ${getDataDuration}ms`);
  
  const overallDuration = Date.now() - overallStart;
  log.info(`\n✅ 爬虫完成!`);
  log.info(`📊 成功抓取: ${results.items.length} 个商品`);
  log.info(`⏱️  [总耗时] ${overallDuration}ms (${(overallDuration / 1000).toFixed(2)}s)`);
  log.info(`📈 平均每个URL耗时: ${(overallDuration / urlList.length).toFixed(2)}ms`);
  
  return results.items as ProductData[];
}

/**
 * 切换爬虫配置预设
 */
function switchPreset(preset: string) {
  currentConfig = getCrawlerConfig(preset);
  log.info(`🔄 已切换到预设: ${preset}`);
}

export {
  runOttoCrawler,
  createOttoCrawler,
  extractOttoData,
  extractText,
  switchPreset,
  currentConfig
};

// 如果直接运行此文件,执行示例
if (import.meta.url === `file://${process.argv[1]}`) {
  const exampleUrls = [
    'https://www.otto.de/p/flieks-etagenbett-kinderbett-hausbett-90x200cm-mit-schoenem-fenster-dach-und-schraegleiter-S03FA0WG/'
  ];
  
  runOttoCrawler(exampleUrls, { preset: 'stealth' })
    .then(results => {
      console.log('\n📦 抓取结果:');
      console.log(JSON.stringify(results, null, 2));
    })
    .catch(error => {
      console.error('❌ 爬虫运行失败:', error);
      process.exit(1);
    });
}