fix craw fail issue1

crawler.py

@@ -27,11 +27,17 @@ class NewsItem:
     sentiment_score: Optional[float] = None
 
 class CnYesNewsCrawler:
-    """鉅亨網新聞爬蟲"""
+    """鉅亨網新聞爬蟲 - 改進版"""
     
     def __init__(self):
         self.base_url = "https://news.cnyes.com"
-        self.session = cloudscraper.create_scraper()
+        self.session = cloudscraper.create_scraper(
+            browser={
+                'browser': 'chrome',
+                'platform': 'windows',
+                'mobile': False
+            }
+        )
         self.ua = UserAgent()
         
         # 新聞分類URL
@@ -44,10 +50,10 @@ class CnYesNewsCrawler:
         self._setup_headers()
         
     def _setup_headers(self):
-        """設置隨機請求頭"""
+        """設置更真實的請求頭"""
         self.session.headers.update({
-            'User-Agent': self.ua.random,
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
             'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8',
             'Accept-Encoding': 'gzip, deflate, br',
             'DNT': '1',
@@ -56,36 +62,49 @@ class CnYesNewsCrawler:
             'Sec-Fetch-Dest': 'document',
             'Sec-Fetch-Mode': 'navigate',
             'Sec-Fetch-Site': 'none',
-            'Cache-Control': 'max-age=0'
+            'Sec-Fetch-User': '?1',
+            'Cache-Control': 'max-age=0',
+            'sec-ch-ua': '"Not_A Brand";v="8", "Chromium";v="120", "Google Chrome";v="120"',
+            'sec-ch-ua-mobile': '?0',
+            'sec-ch-ua-platform': '"Windows"'
         })
     
     def _get_page(self, url: str, retries: int = 3) -> Optional[BeautifulSoup]:
-        """獲取網頁內容"""
+        """獲取網頁內容 - 改進版"""
         for attempt in range(retries):
             try:
-                # 隨機延遲
-                time.sleep(random.uniform(2, 5))
+                # 更長的隨機延遲，模擬人類行為
+                time.sleep(random.uniform(8, 15))
                 
                 # 輪換 User-Agent
-                self.session.headers['User-Agent'] = self.ua.random
+                user_agents = [
+                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
+                    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0'
+                ]
+                self.session.headers['User-Agent'] = random.choice(user_agents)
                 
+                logger.info(f"正在請求: {url}")
                 response = self.session.get(url, timeout=30)
                 
                 if response.status_code == 200:
                     response.encoding = 'utf-8'
-                    return BeautifulSoup(response.content, 'html.parser')
+                    soup = BeautifulSoup(response.content, 'html.parser')
+                    logger.info(f"成功獲取網頁: {url}")
+                    return soup
                 else:
                     logger.warning(f"HTTP {response.status_code} for {url}")
                     
             except Exception as e:
                 logger.error(f"請求失敗 (嘗試 {attempt + 1}/{retries}): {e}")
                 if attempt < retries - 1:
-                    time.sleep(random.uniform(5, 10))
+                    time.sleep(random.uniform(15, 30))
                     
         return None
     
-    def _extract_article_urls(self, category_url: str, max_pages: int = 3) -> List[str]:
-        """從分類頁面提取文章URL"""
+    def _extract_article_urls(self, category_url: str, max_pages: int = 2) -> List[str]:
+        """從分類頁面提取文章URL - 改進版"""
         article_urls = []
         
         for page in range(1, max_pages + 1):
@@ -93,21 +112,35 @@ class CnYesNewsCrawler:
                 if page == 1:
                     url = category_url
                 else:
+                    # 修正分頁URL格式
                     url = f"{category_url}?page={page}"
                 
-                logger.info(f"爬取分類頁面: {url}")
+                logger.info(f"爬取分類頁面 {page}: {url}")
                 soup = self._get_page(url)
                 
                 if not soup:
                     continue
                 
-                # 尋找文章連結
-                links = soup.find_all('a', href=re.compile(r'/news/id/\d+'))
+                # 改進的選擇器，針對鉅亨網的實際結構
+                link_selectors = [
+                    'a[href*="/news/id/"]',
+                    '.news-list a[href*="/news/id/"]',
+                    '.list-item a[href*="/news/id/"]',
+                    '.news-item a[href*="/news/id/"]',
+                    'h3 a[href*="/news/id/"]',
+                    '.title a[href*="/news/id/"]'
+                ]
+                
                 page_urls = []
+                for selector in link_selectors:
+                    links = soup.select(selector)
+                    if links:
+                        logger.info(f"使用選擇器 '{selector}' 找到 {len(links)} 個連結")
+                        break
                 
                 for link in links:
                     href = link.get('href')
-                    if href:
+                    if href and '/news/id/' in href:
                         full_url = urljoin(self.base_url, href)
                         if full_url not in page_urls:
                             page_urls.append(full_url)
@@ -116,27 +149,44 @@ class CnYesNewsCrawler:
                 logger.info(f"第 {page} 頁找到 {len(page_urls)} 篇文章")
                 
                 if not page_urls:
+                    logger.warning(f"第 {page} 頁沒有找到文章，可能遇到反爬蟲機制")
                     break
                     
+                # 頁面間更長延遲
+                if page < max_pages:
+                    time.sleep(random.uniform(20, 40))
+                    
             except Exception as e:
                 logger.error(f"爬取第 {page} 頁時發生錯誤: {e}")
                 continue
         
-        return list(set(article_urls))  # 去重
+        # 去重並限制數量
+        unique_urls = list(set(article_urls))
+        logger.info(f"總共找到 {len(unique_urls)} 篇獨特文章")
+        return unique_urls
     
     def _extract_article_content(self, url: str, category: str) -> Optional[NewsItem]:
-        """提取文章詳細內容"""
+        """提取文章詳細內容 - 改進版"""
         try:
             soup = self._get_page(url)
             if not soup:
                 return None
             
-            # 提取標題
+            # 調試：打印網頁結構的一部分
+            logger.info(f"網頁標題標籤: {[tag.name for tag in soup.find_all(['h1', 'h2', 'h3'])]}")
+            
+            # 改進的標題選擇器
             title_selectors = [
-                'h1.news-title',
                 'h1[class*="title"]',
+                'h1.news-title',
+                'h1.article-title',
                 '.article-header h1',
-                'h1'
+                '.news-header h1',
+                '.content-header h1',
+                'h1',
+                'h2[class*="title"]',
+                '.title h1',
+                '.title h2'
             ]
             
             title = ""
@@ -144,47 +194,90 @@ class CnYesNewsCrawler:
                 title_elem = soup.select_one(selector)
                 if title_elem:
                     title = title_elem.get_text(strip=True)
-                    if title and len(title) > 5:
+                    if title and len(title) > 10:
+                        logger.info(f"使用選擇器 '{selector}' 找到標題: {title[:50]}...")
                         break
             
             if not title:
                 logger.warning(f"無法提取標題: {url}")
+                # 嘗試從頁面標題獲取
+                page_title = soup.find('title')
+                if page_title:
+                    title = page_title.get_text(strip=True).split(' | ')[0]
+                    logger.info(f"從頁面標題獲取: {title[:50]}...")
+            
+            if not title or len(title) < 5:
+                logger.warning(f"標題太短或無法提取: {url}")
                 return None
             
-            # 提取內容
+            # 改進的內容選擇器
             content_selectors = [
-                '.news-content',
                 '.article-content',
+                '.news-content', 
                 '.content-body',
-                '[class*="article-text"]'
+                '.article-body',
+                '.news-body',
+                '.post-content',
+                '[class*="article-text"]',
+                '[class*="content"]',
+                '.article p',
+                '.content p'
             ]
             
             content = ""
             for selector in content_selectors:
-                content_elem = soup.select_one(selector)
-                if content_elem:
+                content_container = soup.select_one(selector)
+                if content_container:
                     # 移除不需要的元素
-                    for unwanted in content_elem.select('script, style, .ad, .advertisement'):
+                    for unwanted in content_container.select('script, style, .ad, .advertisement, .related, .share, .comment'):
                         unwanted.decompose()
                     
-                    paragraphs = content_elem.find_all(['p', 'div'])
+                    # 提取文本段落
+                    paragraphs = content_container.find_all(['p', 'div'], string=True)
                     content_parts = []
+                    
                     for p in paragraphs:
                         text = p.get_text(strip=True)
-                        if text and len(text) > 10:
+                        if text and len(text) > 20 and not any(skip in text.lower() for skip in ['廣告', 'ad', 'advertisement', '分享', 'share']):
                             content_parts.append(text)
                     
                     content = '\n'.join(content_parts)
-                    if content:
+                    if len(content) > 100:
+                        logger.info(f"使用選擇器 '{selector}' 找到內容，長度: {len(content)}")
                         break
             
+            # 如果還是沒有內容，嘗試獲取所有文本
+            if not content or len(content) < 100:
+                logger.warning(f"常規方法無法提取內容，嘗試備用方法: {url}")
+                
+                # 移除不需要的標籤
+                for unwanted in soup.select('script, style, nav, header, footer, .menu, .sidebar, .ad'):
+                    unwanted.decompose()
+                
+                # 尋找包含最多文本的元素
+                all_text_elements = soup.find_all(['p', 'div'], string=True)
+                text_blocks = []
+                
+                for elem in all_text_elements:
+                    text = elem.get_text(strip=True)
+                    if len(text) > 50:
+                        text_blocks.append(text)
+                
+                if text_blocks:
+                    content = '\n'.join(text_blocks[:10])  # 取前10段
+                    logger.info(f"備用方法找到內容，長度: {len(content)}")
+            
             if not content or len(content) < 50:
-                logger.warning(f"內容太短或無法提取: {url}")
+                logger.warning(f"內容太短或無法提取: {url}, 內容長度: {len(content)}")
+                logger.debug(f"網頁HTML結構預覽: {str(soup)[:500]}...")
                 return None
             
             # 提取發布時間
             published_date = self._extract_publish_date(soup)
             
+            # 清理內容
+            content = self._clean_content(content)
+            
             # 創建新聞項目
             news_item = NewsItem(
                 title=title,
@@ -195,32 +288,55 @@ class CnYesNewsCrawler:
                 published_date=published_date
             )
             
-            logger.info(f"成功提取文章: {title[:50]}...")
+            logger.info(f"成功提取文章: {title[:50]}... (內容長度: {len(content)})")
             return news_item
             
         except Exception as e:
             logger.error(f"提取文章內容時發生錯誤 {url}: {e}")
             return None
     
+    def _clean_content(self, content: str) -> str:
+        """清理內容"""
+        # 移除多餘空白
+        content = re.sub(r'\s+', ' ', content)
+        
+        # 移除特殊字符
+        content = re.sub(r'[^\u4e00-\u9fff\u3400-\u4dbf\w\s.,!?()（），。！？：；「」『』]', '', content)
+        
+        # 移除重複句子
+        sentences = content.split('。')
+        unique_sentences = []
+        for sentence in sentences:
+            if sentence.strip() and sentence.strip() not in unique_sentences:
+                unique_sentences.append(sentence.strip())
+        
+        return '。'.join(unique_sentences)
+    
     def _extract_publish_date(self, soup: BeautifulSoup) -> datetime:
-        """提取發布時間"""
+        """提取發布時間 - 改進版"""
         time_selectors = [
             'time[datetime]',
             '.publish-time',
             '.news-time',
-            '[class*="time"]'
+            '.article-time',
+            '[class*="time"]',
+            '[class*="date"]',
+            'meta[property="article:published_time"]',
+            'meta[name="pubdate"]'
         ]
         
         for selector in time_selectors:
             time_elem = soup.select_one(selector)
             if time_elem:
-                datetime_attr = time_elem.get('datetime')
+                # 檢查datetime屬性
+                datetime_attr = time_elem.get('datetime') or time_elem.get('content')
                 if datetime_attr:
                     try:
                         return datetime.fromisoformat(datetime_attr.replace('Z', '+00:00')).replace(tzinfo=None)
                     except:
                         pass
                 
+                # 檢查文本內容
                 time_text = time_elem.get_text(strip=True)
                 parsed_time = self._parse_time_text(time_text)
                 if parsed_time:
@@ -229,12 +345,14 @@ class CnYesNewsCrawler:
         return datetime.now()
     
     def _parse_time_text(self, time_text: str) -> Optional[datetime]:
-        """解析時間文字"""
+        """解析時間文字 - 改進版"""
         patterns = [
             r'(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})',
             r'(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2})',
             r'(\d{4})/(\d{2})/(\d{2})\s+(\d{2}):(\d{2})',
-            r'(\d{4})-(\d{2})-(\d{2})'
+            r'(\d{4})-(\d{2})-(\d{2})',
+            r'(\d{4})年(\d{1,2})月(\d{1,2})日\s*(\d{1,2}):(\d{2})',
+            r'(\d{4})年(\d{1,2})月(\d{1,2})日'
         ]
         
         for pattern in patterns:
@@ -255,8 +373,8 @@ class CnYesNewsCrawler:
         
         return None
     
-    def crawl_category(self, category: str, max_articles: int = 20) -> List[NewsItem]:
-        """爬取指定分類的新聞"""
+    def crawl_category(self, category: str, max_articles: int = 10) -> List[NewsItem]:
+        """爬取指定分類的新聞 - 減少數量避免被封"""
         if category not in self.categories:
             logger.error(f"無效的分類: {category}")
             return []
@@ -265,13 +383,13 @@ class CnYesNewsCrawler:
         
         # 獲取文章URL列表
         category_url = self.categories[category]
-        article_urls = self._extract_article_urls(category_url)
+        article_urls = self._extract_article_urls(category_url, max_pages=2)
         
         if not article_urls:
             logger.warning(f"未找到 {category} 分類的文章URL")
             return []
         
-        # 限制文章數量
+        # 限制文章數量，避免被封
         if len(article_urls) > max_articles:
             article_urls = article_urls[:max_articles]
         
@@ -284,8 +402,8 @@ class CnYesNewsCrawler:
                 if article:
                     articles.append(article)
                 
-                # 隨機延遲
-                time.sleep(random.uniform(3, 8))
+                # 更長的隨機延遲，模擬人類閱讀
+                time.sleep(random.uniform(15, 30))
                 
             except Exception as e:
                 logger.error(f"處理文章時發生錯誤 {url}: {e}")
@@ -294,8 +412,8 @@ class CnYesNewsCrawler:
         logger.info(f"{category} 分類爬取完成，共 {len(articles)} 篇文章")
         return articles
     
-    def crawl_all_categories(self, max_articles_per_category: int = 15) -> Dict[str, List[NewsItem]]:
-        """爬取所有分類的新聞"""
+    def crawl_all_categories(self, max_articles_per_category: int = 8) -> Dict[str, List[NewsItem]]:
+        """爬取所有分類的新聞 - 減少數量"""
         results = {}
         
         for category in self.categories.keys():
@@ -304,8 +422,8 @@ class CnYesNewsCrawler:
                 articles = self.crawl_category(category, max_articles_per_category)
                 results[category] = articles
                 
-                # 分類間延遲
-                time.sleep(random.uniform(10, 20))
+                # 分類間更長延遲
+                time.sleep(random.uniform(60, 120))
                 
             except Exception as e:
                 logger.error(f"爬取 {category} 分類時發生錯誤: {e}")