src/batch-crawler.ts

import { runOttoCrawler } from './otto-crawler.js';
import { ProductData } from './productData.js';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

/**
 * 从文件读取 URL 列表11
 */
function readUrlsFromFile(filePath: string): string[] {
  try {
    const content = fs.readFileSync(filePath, 'utf-8');
    return content
      .split('\n')
      .map(line => line.trim())
      .filter(line => line && !line.startsWith('#'));
  } catch (error: any) {
    console.error(`读取文件失败: ${filePath}`, error.message);
    return [];
  }
}

/**
 * 保存结果到 JSON 文件
 */
function saveResultsToJson(results: ProductData[], outputPath: string = './results/otto-products.json'): void {
  try {
    const dir = path.dirname(outputPath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }
    
    fs.writeFileSync(outputPath, JSON.stringify(results, null, 2), 'utf-8');
    console.log(`💾 结果已保存到: ${outputPath}`);
  } catch (error: any) {
    console.error('保存结果失败:', error.message);
  }
}

/**
 * 保存结果到 CSV 文件
 */
function saveResultsToCsv(results: ProductData[], outputPath: string = './results/otto-products.csv'): void {
  try {
    const dir = path.dirname(outputPath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }
    
    if (results.length === 0) {
      console.log('没有数据可保存');
      return;
    }
    
    // 获取所有字段名
    const fields = Object.keys(results[0]);
    
    // 创建 CSV 内容1
    const csvContent = [
      fields.join(','),
      ...results.map(product => 
        fields.map(field => {
          const value = String(product[field] || '');
          return `"${value.replace(/"/g, '""')}"`;
        }).join(',')
      )
    ].join('\n');
    
    fs.writeFileSync(outputPath, csvContent, 'utf-8');
    console.log(`💾 CSV 结果已保存到: ${outputPath}`);
  } catch (error: any) {
    console.error('保存 CSV 失败:', error.message);
  }
}

/**
 * 批量爬取主函数
 */
async function batchCrawl(options: {
  urls?: string[];
  file?: string;
  preset?: string;
  outputFormat?: 'json' | 'csv' | 'both';
  outputFile?: string;
} = {}) {
  let urls: string[] = [];
  
  // 从文件读取或从参数获取 URLs
  if (options.file) {
    console.log(`📄 从文件读取 URLs: ${options.file}`);
    urls = readUrlsFromFile(options.file);
  } else if (options.urls && options.urls.length > 0) {
    urls = options.urls;
  } else {
    console.error('❌ 请提供 URL 列表或文件路径');
    process.exit(1);
  }
  
  if (urls.length === 0) {
    console.error('❌ 没有有效的 URL');
    process.exit(1);
  }
  
  console.log(`📋 待爬取 URL 数量: ${urls.length}`);
  console.log(`🔧 配置预设: ${options.preset || 'stable'}`);
  
  try {
    // 执行爬取
    const results = await runOttoCrawler(urls, { 
      preset: options.preset || 'stable' 
    });
    
    console.log(`\n✅ 爬取完成!`);
    console.log(`📊 成功抓取: ${results.length} 个商品\n`);
    
    // 保存结果
    const outputFormat = options.outputFormat || 'json';
    const outputFile = options.outputFile || './results/otto-products';
    
    if (outputFormat === 'json' || outputFormat === 'both') {
      saveResultsToJson(results, `${outputFile}.json`);
    }
    
    if (outputFormat === 'csv' || outputFormat === 'both') {
      saveResultsToCsv(results, `${outputFile}.csv`);
    }
    
    return results;
  } catch (error: any) {
    console.error('\n❌ 爬取失败:', error.message);
    process.exit(1);
  }
}

// 如果直接运行此文件
if (import.meta.url === `file://${process.argv[1]}`) {
  const args = process.argv.slice(2);
  const options: any = {};
  
  // 解析命令行参数
  for (let i = 0; i < args.length; i++) {
    switch (args[i]) {
      case '--file':
      case '-f':
        options.file = args[++i];
        break;
      case '--preset':
      case '-p':
        options.preset = args[++i];
        break;
      case '--format':
        options.outputFormat = args[++i];
        break;
      case '--output':
      case '-o':
        options.outputFile = args[++i];
        break;
      default:
        if (!args[i].startsWith('-')) {
          if (!options.urls) options.urls = [];
          options.urls.push(args[i]);
        }
    }
  }
  
  batchCrawl(options);
}

export { batchCrawl, readUrlsFromFile, saveResultsToJson, saveResultsToCsv };