🚀 Auto-deploy: 8968db766ef462f994117cc8e723669bb87429fb

package.json

@@ -7,7 +7,7 @@
     "start": "tsx quick-start.ts",
     "serve": "tsx watch server.ts",
     "start:prod": "tsx server.ts",
-    "crawl:otto": "tsx src/otto-crawler.ts",
+    "crawl:otto": "tsx src/-crawler.ts",
     "crawl:amazon": "tsx src/amazon-crawler.ts",
     "crawl:batch": "tsx src/batch-crawler.ts",
     "quick": "tsx quick-start.ts",

src/anti-detection.ts

@@ -0,0 +1,121 @@
+import { Page } from 'playwright';
+
+/**
+ * 注入全面的反自动化检测脚本
+ */
+export async function injectAntiDetectionScripts(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    // 覆盖navigator.webdriver属性
+    Object.defineProperty(navigator, 'webdriver', {
+      get: () => undefined,
+    });
+    
+    // 移除Chrome特有属性
+    (window as any).chrome = {
+      runtime: {},
+    };
+    
+    // 覆盖plugins和languages
+    Object.defineProperty(navigator, 'plugins', {
+      get: () => [1, 2, 3, 4, 5] as any,
+    });
+    
+    Object.defineProperty(navigator, 'languages', {
+      get: () => ['en-US', 'en'],
+    });
+    
+    // 隐藏WebDriver属性的其他方式
+    if ('_Selenium_IDE_Recorder' in window) {
+      delete window['_Selenium_IDE_Recorder'];
+    }
+    
+    // 覆盖userAgent中的Headless标识
+    const originalUserAgent = navigator.userAgent;
+    Object.defineProperty(navigator, 'userAgent', {
+      value: originalUserAgent.replace('HeadlessChrome', 'Chrome'),
+    });
+    
+    // 隐藏automation相关属性
+    Object.defineProperty(navigator, 'permissions', {
+      value: {
+        ...navigator.permissions,
+        query: new Proxy(navigator.permissions.query, {
+          apply(target, thisArg, args) {
+            return target.apply(thisArg, args).catch(() => ({
+              state: 'granted',
+              onchange: null,
+              addEventListener: () => {},
+              removeEventListener: () => {},
+              dispatchEvent: () => true,
+            }));
+          }
+        })
+      }
+    });
+  });
+}
+
+/**
+ * 设置真实的浏览器视窗尺寸
+ */
+export async function setRealisticViewport(page: Page): Promise<void> {
+  // 设置常见的桌面分辨率
+  const viewports = [
+    { width: 1920, height: 1080 },
+    { width: 1366, height: 768 },
+    { width: 1536, height: 864 },
+    { width: 1440, height: 900 }
+  ];
+  const randomViewport = viewports[Math.floor(Math.random() * viewports.length)];
+  await page.setViewportSize(randomViewport);
+}
+
+/**
+ * 模拟真实用户行为
+ */
+export async function simulateHumanBehavior(page: Page): Promise<void> {
+  try {
+    // 设置真实视窗
+    await setRealisticViewport(page);
+    
+    // 随机鼠标移动
+    const dimensions = await page.evaluate(() => ({
+      width: document.documentElement.clientWidth,
+      height: document.documentElement.clientHeight
+    }));
+    
+    if (dimensions.width > 0 && dimensions.height > 0) {
+      // 多次随机移动模拟真实浏览
+      for (let i = 0; i < 2 + Math.floor(Math.random() * 3); i++) {
+        const x = Math.floor(Math.random() * dimensions.width * 0.8) + 50;
+        const y = Math.floor(Math.random() * dimensions.height * 0.8) + 50;
+        await page.mouse.move(x, y);
+        await page.waitForTimeout(200 + Math.random() * 300);
+      }
+    }
+    
+    // 随机滚动
+    const scrollHeight = await page.evaluate(() => document.body.scrollHeight);
+    if (scrollHeight > 0) {
+      const scrollPositions = [0.1, 0.3, 0.5, 0.7];
+      const randomPos = scrollPositions[Math.floor(Math.random() * scrollPositions.length)];
+      const randomScroll = Math.min(scrollHeight * randomPos, 1200);
+      await page.evaluate((height) => window.scrollTo(0, height), randomScroll);
+    }
+    
+    // 随机等待时间
+    const randomWait = Math.floor(Math.random() * 1000) + 800;
+    await page.waitForTimeout(randomWait);
+    
+  } catch (error) {
+    // 忽略模拟行为的错误，不影响主要功能
+  }
+}
+
+/**
+ * 添加随机延迟
+ */
+export async function addRandomDelay(page: Page, minMs: number = 1500, maxMs: number = 4000): Promise<void> {
+  const delay = Math.floor(Math.random() * (maxMs - minMs)) + minMs;
+  await page.waitForTimeout(delay);
+}

src/config.ts

@@ -83,11 +83,11 @@ const PRESETS: Record<string, Partial<CrawlerConfig>> = {
     }
   },
   stealth: {
-    maxConcurrency: 3,
-    requestHandlerTimeoutSecs: 180,
-    maxRequestRetries: 1,
-    pageLoadWaitMs: 500,
-    useProxy: true, // 启用代理
+    maxConcurrency: 2, // 进一步降低并发数
+    requestHandlerTimeoutSecs: 240, // 延长超时时间
+    maxRequestRetries: 2, // 增加重试次数
+    pageLoadWaitMs: 800, // 增加页面等待时间
+    useProxy: false, // 启用代理
     launchOptions: { 
       headless: 'new',
       args: [
@@ -105,7 +105,18 @@ const PRESETS: Record<string, Partial<CrawlerConfig>> = {
         '--mute-audio',
         '--no-first-run',
         '--no-pings',
-        '--disable-features=IsolateOrigins,site-per-process'
+        '--disable-features=IsolateOrigins,site-per-process',
+        // 新增反检测参数
+        '--disable-blink-features=AutomationControlled',
+        '--disable-automation',
+        '--disable-web-security',
+        '--allow-running-insecure-content',
+        '--disable-features=VizDisplayCompositor',
+        '--disable-features=ImprovedCookieControls,SameSiteByDefaultCookies,CookiesWithoutSameSiteMustBeSecure',
+        '--disable-features=WebRtcHideLocalIpsWithMdns',
+        '--disable-features=WebContentsForceDark',
+        '--disable-features=AutofillEnableAccountWalletStorage',
+        '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
       ]
     }
   }

src/otto-crawler.ts

@@ -3,6 +3,8 @@ import { Page } from 'playwright';
 import { getCrawlerConfig, CrawlerConfig } from './config.js';
 import { ProductData } from './productData.js';
 import { createProxyConfiguration } from './proxy-config.js';
+// 导入反检测模块
+import { injectAntiDetectionScripts, simulateHumanBehavior, addRandomDelay } from './anti-detection.js';
 
 // 默认配置
 let currentConfig: CrawlerConfig = getCrawlerConfig();
@@ -93,13 +95,16 @@ function createOttoCrawler(options: { preset?: string } & Partial<CrawlerConfig>
     requestHandlerTimeoutSecs: config.requestHandlerTimeoutSecs,
     maxRequestRetries: config.maxRequestRetries,
     navigationTimeoutSecs: 120, // 导航超时120秒
-    // 使用从ip.txt加载的代理配置
+    // 使用 Decodo 代理配置
     proxyConfiguration: proxyConfiguration,
     launchContext: {
       launchOptions
     },
     preNavigationHooks: [
       async ({ page }) => {
+        // 注入反检测脚本
+        await injectAntiDetectionScripts(page);
+        
         // 拦截并 abort 非必要的资源类型
         await page.route('**/*', (route, request) => {
           const resourceType = request.resourceType();
@@ -116,17 +121,21 @@ function createOttoCrawler(options: { preset?: string } & Partial<CrawlerConfig>
       const startTime = Date.now();
       log.info(`🔄 开始处理: ${request.url}`);
       
+      // 添加随机延迟，模拟真实用户行为
+      await addRandomDelay(page, 1000, 3000);
+      
       // 1. 等待 DOM 加载完成
       const domStartTime = Date.now();
       await page.waitForLoadState('domcontentloaded');
       const domLoadTime = Date.now() - domStartTime;
       log.info(`⏱️ DOM 加载耗时: ${domLoadTime}ms`);
       
-      // 2. 智能等待:等待页面标题或价格等关键元素出现
+      // 2. 模拟真实用户行为
+      await simulateHumanBehavior(page);
+      
+      // 3. 智能等待:等待页面标题或价格等关键元素出现
       const waitStartTime = Date.now();
       const waitForSelectors = [
-        // ...currentConfig.selectors.title || [],
-        // ...currentConfig.selectors.price || []
         'span.js_pdp_cr-rating--review-count'
       ];
       
@@ -151,7 +160,7 @@ function createOttoCrawler(options: { preset?: string } & Partial<CrawlerConfig>
         log.warning('⚠️ 未检测到关键元素,但将继续尝试提取数据');
       }
       
-      // 3. 短暂等待让动态内容渲染(可选,根据实际情况调整)
+      // 4. 短暂等待让动态内容渲染(可选,根据实际情况调整)
       await page.waitForTimeout(500);
       
       const productData: ProductData = {
@@ -252,10 +261,10 @@ export {
 // 如果直接运行此文件,执行示例
 if (import.meta.url === `file://${process.argv[1]}`) {
   const exampleUrls = [
-    'https://www.otto.de/p/samsung-galaxy-s23-sm-s911b-dual-sim-256-gb-phantom-black-1487828507/'
+    'https://www.otto.de/p/flieks-etagenbett-kinderbett-hausbett-90x200cm-mit-schoenem-fenster-dach-und-schraegleiter-S03FA0WG/'
   ];
   
-  runOttoCrawler(exampleUrls, { preset: 'stable' })
+  runOttoCrawler(exampleUrls, { preset: 'stealth' })
     .then(results => {
       console.log('\n📦 抓取结果:');
       console.log(JSON.stringify(results, null, 2));

src/proxy-config.ts

@@ -1,47 +1,27 @@
 import { ProxyConfiguration } from 'crawlee';
-import * as fs from 'fs';
-import * as path from 'path';
 
-// 读取IP代理列表
-export function loadProxyUrls(): string[] {
-  try {
-    const ipFilePath = path.join(process.cwd(), 'storage', 'ip.txt');
-    if (!fs.existsSync(ipFilePath)) {
-      console.warn('⚠️ IP代理文件不存在:', ipFilePath);
-      return [];
-    }
-    
-    const content = fs.readFileSync(ipFilePath, 'utf-8');
-    const lines = content.split('\n').map(line => line.trim()).filter(line => line.length > 0);
-    
-    // 转换为完整的代理URL格式 (假设是HTTP代理，无认证)
-    const proxyUrls = lines.map(ipPort => {
-      // 检查是否已经是完整的URL格式
-      if (ipPort.startsWith('http://') || ipPort.startsWith('https://')) {
-        return ipPort;
-      }
-      // 否则假设是IP:PORT格式，转换为http://IP:PORT
-      return `http://${ipPort}`;
-    });
-    
-    console.log(`✅ 成功加载 ${proxyUrls.length} 个代理IP`);
-    return proxyUrls;
-  } catch (error) {
-    console.error('❌ 加载代理IP失败:', error);
-    return [];
-  }
+// Dataify 动态住宅代理配置
+const PROXY_HOST = 'eu.dataify.top';
+const PROXY_PORT = '6600';
+const PROXY_USERNAME = 'gqFOVXhf9yd2-r-de';
+const PROXY_PASSWORD = 'm9E0BDFM';
+
+// 创建代理URL
+function getProxyUrl(): string {
+  return `https://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}:${PROXY_PORT}`;
 }
 
-// 创建代理配置
+// 创建代理配置（Dataify 动态代理，每次请求自动轮换IP）
 export function createProxyConfiguration(): ProxyConfiguration | undefined {
-  const proxyUrls = loadProxyUrls();
-  
-  if (proxyUrls.length === 0) {
-    console.warn('⚠️ 未找到可用的代理IP，将不使用代理');
-    return undefined;
-  }
-  
+  const proxyUrl = getProxyUrl();
+  console.log(`✅ 使用 Dataify 动态代理: ${PROXY_HOST}:${PROXY_PORT}`);
+
   return new ProxyConfiguration({
-    proxyUrls: proxyUrls
+    proxyUrls: [proxyUrl],
   });
-}
+}
+
+// 保留向后兼容的函数
+export function loadProxyUrls(): string[] {
+  return [getProxyUrl()];
+}