src/amazon-crawler.ts

import { PlaywrightCrawler, ProxyConfiguration, Dataset, log } from 'crawlee';
import { Page } from 'playwright';
import { getCrawlerConfig, CrawlerConfig } from './config.js';

// Import ProductData type from otto-crawler to maintain consistency
import { ProductData } from './productData.js';

// Import proxy configuration
import { createProxyConfiguration } from './proxy-config.js';

// Default configuration for Amazon crawler
let currentConfig: CrawlerConfig = getCrawlerConfig();

/**
 * Amazon-specific selectors configuration
 */
const AMAZON_SELECTORS = {
  // Brand/Title selectors
  brand: [
    '#bylineInfo',
    '#brand',
    '.a-link-normal[tabindex="-1"]'
  ],
  title: [
    '#productTitle',
    'h1[data-testid="product-title"]',
    '.a-size-large.product-title-word-break'
  ],
  price: [
    '.a-price-whole',
    '#priceblock_ourprice',
    '.a-offscreen',
    '[data-testid="price"] .a-price-whole'
  ],
  originalPrice: [
    '.a-price.a-text-price span.a-offscreen',
    '#priceblock_dealprice',
    '.a-price-was .a-offscreen'
  ],
  rating: [
    '#acrPopover',
    '.a-icon-alt',
    '[data-hook="rating-out-of-text"]'
  ],
  reviews: [
    '#acrCustomerReviewText',
    '[data-hook="total-review-count"]',
    '.a-link-normal[href*="reviews"]'
  ],
  imageUrl: [
    '#landingImage',
    '#imgTagWrapperId img',
    '.a-dynamic-image[data-a-dynamic-image]'
  ]
};

/**
 * Extract text content from page using multiple selectors
 */
async function extractText(page: Page, ...selectors: string[]): Promise<string | null> {
  for (const selector of selectors) {
    try {
      const locator = page.locator(selector).first();
      const count = await locator.count();
      
      if (count > 0) {
        const text = await locator.textContent();
        if (text && text.trim()) {
          return text.trim();
        }
      }
    } catch (error: any) {
      log.debug(`Amazon文本选择器失败: ${selector}`, error.message);
    }
  }
  return null;
}

/**
 * Extract image URL from page
 */
async function extractImageUrl(page: Page, ...selectors: string[]): Promise<string | null> {
  for (const selector of selectors) {
    try {
      const locator = page.locator(selector).first();
      const count = await locator.count();
      
      if (count > 0) {
        // Try different attribute extraction methods
        const src = await locator.getAttribute('src');
        if (src && src.trim()) {
          return src.trim();
        }
        
        const dataSrc = await locator.getAttribute('data-old-hires');
        if (dataSrc && dataSrc.trim()) {
          return dataSrc.trim();
        }
        
        const dataDynamic = await locator.getAttribute('data-a-dynamic-image');
        if (dataDynamic && dataDynamic.trim()) {
          // Parse JSON from data-a-dynamic-image attribute
          try {
            const imageData = JSON.parse(dataDynamic);
            const urls = Object.keys(imageData);
            if (urls.length > 0) {
              return urls[0];
            }
          } catch (e) {
            // If JSON parsing fails, return as-is
            return dataDynamic.trim();
          }
        }
      }
    } catch (error: any) {
      log.debug(`Amazon图片选择器失败: ${selector}`, error.message);
    }
  }
  return null;
}

/**
 * Extract Amazon product details data
 */
async function extractAmazonData(page: Page, product: ProductData = {} as ProductData): Promise<ProductData> {
  try {
    // Extract brand
    const brand = await extractText(page, ...AMAZON_SELECTORS.brand);
    if (brand) product.brand = brand;
    
    // Extract title
    const title = await extractText(page, ...AMAZON_SELECTORS.title);
    if (title) product.title = title;
    
    // Extract price
    const price = await extractText(page, ...AMAZON_SELECTORS.price);
    if (price) product.price = price;

    // Extract original price (for deals)
    const originalPrice = await extractText(page, ...AMAZON_SELECTORS.originalPrice);
    if (originalPrice) product.originalPrice = originalPrice;
    
    // Extract rating
    const rating = await extractText(page, ...AMAZON_SELECTORS.rating);
    if (rating) {
      // Clean up rating text (e.g., "4.5 out of 5 stars" -> "4.5")
      const ratingMatch = rating.match(/(\d+\.?\d*)/);
      product.rating = ratingMatch ? ratingMatch[1] : rating;
    }
    
    // Extract reviews count
    const reviews = await extractText(page, ...AMAZON_SELECTORS.reviews);
    if (reviews) {
      // Clean up reviews text (e.g., "1,234 ratings" -> "1,234")
      const reviewsMatch = reviews.match(/[\d,]+/);
      product.reviews = reviewsMatch ? reviewsMatch[0] : reviews;
    }
    
    // Extract image URL
    const imageUrl = await extractImageUrl(page, ...AMAZON_SELECTORS.imageUrl);
    if (imageUrl) product.imageUrl = imageUrl;
    
    log.info(`✓ 成功提取Amazon商品数据:`, {
      brand: product.brand || 'N/A',
      title: product.title?.substring(0, 50) + '...' || 'N/A',
      price: product.price || 'N/A',
      rating: product.rating || 'N/A',
      reviews: product.reviews || 'N/A'
    });
    
    return product;
  } catch (error: any) {
    log.warning('Amazon数据提取失败:', error.message);
    return product;
  }
}

/**
 * Create Amazon product crawler instance
 */
function createAmazonCrawler(options: { preset?: string } & Partial<CrawlerConfig> = {}) {
  // Merge configuration
  const config = getCrawlerConfig(options.preset, options);
  currentConfig = config; // Update current configuration
  
  // Handle headless type conversion
  const launchOptions: any = {
    ...config.launchOptions,
    headless: config.launchOptions.headless === 'new' ? true : config.launchOptions.headless,
    // Increase browser-level timeout
    timeout: 120000, // 120 seconds
  };
  
  // Get proxy configuration
  const proxyConfiguration = createProxyConfiguration();
  
  const crawler = new PlaywrightCrawler({
    maxConcurrency: config.maxConcurrency,
    requestHandlerTimeoutSecs: config.requestHandlerTimeoutSecs,
    maxRequestRetries: config.maxRequestRetries,
    navigationTimeoutSecs: 120, // Navigation timeout 120 seconds
    // Use proxy configuration from ip.txt
    proxyConfiguration: proxyConfiguration,
    launchContext: {
      launchOptions
    },
    preNavigationHooks: [
      async ({ page }) => {
        // Intercept and abort non-essential resource types
        await page.route('**/*', (route, request) => {
          const resourceType = request.resourceType();
          // Block images, fonts, media files, but keep stylesheets for proper layout
          if (['image', 'font', 'media', 'stylesheet'].includes(resourceType)) {
            // For Amazon, we might want to allow some images for verification
            // But block most to speed up loading
            if (resourceType === 'image' && !request.url().includes('images-amazon.com')) {
              route.abort();
            } else if (['font', 'media'].includes(resourceType)) {
              route.abort();
            } else {
              route.continue();
            }
          } else {
            route.continue();
          }
        });
      }
    ],
    async requestHandler({ request, page, log }) {
      const startTime = Date.now();
      log.info(`🔄 开始处理Amazon商品: ${request.url}`);
      
      // 1. Wait for DOM to load
      const domStartTime = Date.now();
      await page.waitForLoadState('domcontentloaded');
      const domLoadTime = Date.now() - domStartTime;
      log.info(`⏱️ DOM加载耗时: ${domLoadTime}ms`);
      
      // 2. Smart wait: wait for key elements to appear
      const waitStartTime = Date.now();
      const waitForSelectors = [
        '#productTitle',
        '.a-price-whole',
        '#bylineInfo'
      ];
      
      let pageLoaded = false;
      let matchedSelector = '';
      for (const selector of waitForSelectors) {
        try {
          await page.waitForSelector(selector, { state: 'visible', timeout: 8000 });
          pageLoaded = true;
          matchedSelector = selector;
          log.debug(`✓ 找到关键元素: ${selector}`);
          break;
        } catch (e) {
          continue;
        }
      }
      const waitTime = Date.now() - waitStartTime;
      log.info(`⏱️ 元素等待耗时: ${waitTime}ms${pageLoaded ? ` (${matchedSelector})` : ' (未匹配)'}`);
      
      if (!pageLoaded) {
        log.warning('⚠️ 未检测到关键元素，但将继续尝试提取数据');
      }
      
      // 3. Brief wait for dynamic content rendering
      await page.waitForTimeout(500);
      
      const productData: ProductData = {
        url: request.url,
        crawledAt: new Date().toISOString()
      };
      
      const extractStartTime = Date.now();
      const extractedData = await extractAmazonData(page, productData);
      const extractTime = Date.now() - extractStartTime;
      log.info(`⏱️ 数据提取耗时: ${extractTime}ms`);
      
      await Dataset.pushData(extractedData);
      
      const totalTime = Date.now() - startTime;
      log.info(`✅ 完成处理: ${request.url} (总耗时: ${totalTime}ms)`);
    },
    // Error handling with correct parameter signature
    async failedRequestHandler({ request, log }, error) {
      log.error(`Amazon请求失败 (重试 ${request.retryCount}/${config.maxRequestRetries}): ${request.url}`, { 
        message: (error as Error).message 
      });
    },
    errorHandler({ log }, error) {
      log.error('Amazon爬虫全局错误:', { message: (error as Error).message });
    }
  });
  
  return crawler;
}

/**
 * Run Amazon product crawler
 */
async function runAmazonCrawler(urls: string | string[], options: { preset?: string } & Partial<CrawlerConfig> = {}): Promise<ProductData[]> {
  const overallStart = Date.now();
  const urlList = Array.isArray(urls) ? urls : [urls];
  
  log.info('🚀 开始Amazon商品爬虫...');
  log.info(`📋 待爬取URL数量: ${urlList.length}`);
  log.info(`🔧 配置预设: ${options.preset || 'default'}`);
  
  // Step 1: Create crawler instance
  const createStart = Date.now();
  const crawler = createAmazonCrawler(options);
  const createDuration = Date.now() - createStart;
  log.info(`⏱️ [步骤1] 创建爬虫实例耗时: ${createDuration}ms`);
  
  // Step 2: Add requests to queue
  const addRequestsStart = Date.now();
  await crawler.addRequests(urlList.map(url => ({
    url,
    userData: { label: 'DETAIL' }
  })));
  const addRequestsDuration = Date.now() - addRequestsStart;
  log.info(`⏱️ [步骤2] 添加请求到队列耗时: ${addRequestsDuration}ms`);
  
  // Step 3: Execute crawling
  log.info('🔄 开始执行爬取任务...');
  const runStart = Date.now();
  await crawler.run();
  const runDuration = Date.now() - runStart;
  log.info(`⏱️ [步骤3] 执行爬取任务耗时: ${runDuration}ms (${(runDuration / 1000).toFixed(2)}s)`);
  
  // Step 4: Get results
  const getDataStart = Date.now();
  const results = await Dataset.getData();
  const getDataDuration = Date.now() - getDataStart;
  log.info(`⏱️ [步骤4] 获取结果数据耗时: ${getDataDuration}ms`);
  
  const overallDuration = Date.now() - overallStart;
  log.info(`\n✅ 爬虫完成!`);
  log.info(`📊 成功抓取: ${results.items.length} 个商品`);
  log.info(`⏱️ [总耗时] ${overallDuration}ms (${(overallDuration / 1000).toFixed(2)}s)`);
  log.info(`📈 平均每个URL耗时: ${(overallDuration / urlList.length).toFixed(2)}ms`);
  
  return results.items as ProductData[];
}

/**
 * Switch crawler configuration preset
 */
function switchPreset(preset: string) {
  currentConfig = getCrawlerConfig(preset);
  log.info(`🔄 已切换到预设: ${preset}`);
}

export {
  runAmazonCrawler,
  createAmazonCrawler,
  extractAmazonData,
  extractText,
  extractImageUrl,
  switchPreset,
  currentConfig,
  AMAZON_SELECTORS
};

// If run directly, execute example
if (import.meta.url === `file://${process.argv[1]}`) {
  const exampleUrls = [
    'https://www.amazon.com/dp/B08N5WRWNW', // Example product URL
    'https://www.amazon.de/dp/B08N5WRWNW'   // German Amazon example
  ];
  
  runAmazonCrawler(exampleUrls, { preset: 'stable' })
    .then(results => {
      console.log('\n📦 抓取结果:');
      console.log(JSON.stringify(results, null, 2));
    })
    .catch(error => {
      console.error('❌ 爬虫运行失败:', error);
      process.exit(1);
    });
}