Update crawler.py

crawler.py

@@ -27,7 +27,7 @@ class NewsItem:
     sentiment_score: Optional[float] = None
 
 class CnYesNewsCrawler:
-    """鉅亨網新聞爬蟲 - 修正URL版"""
+    """鉅亨網新聞爬蟲 - 無限制版"""
     
     def __init__(self, sentiment_analyzer=None, database=None):
         self.base_url = "https://news.cnyes.com"
@@ -122,8 +122,8 @@ class CnYesNewsCrawler:
                     
         return None
     
-    def _extract_article_urls(self, category_url: str, max_pages: int = 2) -> List[str]:
-        """從分類頁面提取文章URL"""
+    def _extract_article_urls(self, category_url: str, max_pages: int = 3) -> List[str]:
+        """從分類頁面提取文章URL - 增加頁數"""
         article_urls = []
         
         for page in range(1, max_pages + 1):
@@ -349,8 +349,8 @@ class CnYesNewsCrawler:
         
         return None
     
-    def crawl_category(self, category: str, max_articles: int = 10) -> List[NewsItem]:
-        """爬取指定分類的新聞"""
+    def crawl_category(self, category: str, max_articles: int = None) -> List[NewsItem]:
+        """爬取指定分類的新聞 - 移除文章數量限制"""
         if category not in self.categories:
             logger.error(f"無效的分類: {category}")
             return []
@@ -360,18 +360,27 @@ class CnYesNewsCrawler:
         
         # 獲取文章URL列表
         category_url = self.categories[category]
-        article_urls = self._extract_article_urls(category_url, max_pages=2)
+        article_urls = self._extract_article_urls(category_url, max_pages=3)  # 增加到3頁
         
         if not article_urls:
             self._notify_progress(f"⚠️ 未找到 {category_name} 分類的文章URL")
             return []
         
-        # 限制文章數量
-        if len(article_urls) > max_articles:
-            article_urls = article_urls[:max_articles]
+        # **關鍵修正：不限制文章數量**
+        total_articles = len(article_urls)
+        if max_articles and max_articles > 0:
+            # 只有在明確指定max_articles時才限制
+            if len(article_urls) > max_articles:
+                article_urls = article_urls[:max_articles]
+                self._notify_progress(f"⚠️ 限制處理文章數量為 {max_articles} 篇")
+        
+        self._notify_progress(f"📊 將處理 {len(article_urls)} 篇文章（共找到 {total_articles} 篇）")
         
         # 提取文章內容並即時分析存檔
         articles = []
+        success_count = 0
+        error_count = 0
+        
         for i, url in enumerate(article_urls, 1):
             try:
                 self._notify_progress(f"📖 處理 {category_name} 文章 {i}/{len(article_urls)}: 正在提取內容...")
@@ -406,40 +415,46 @@ class CnYesNewsCrawler:
                                 inserted, _ = self.database.insert_news([db_article])
                                 if inserted > 0:
                                     self._notify_progress(f"💾 已保存 {category_name} 文章: {article.title[:30]}... (情緒: {article.sentiment})")
+                                    success_count += 1
                                 else:
                                     self._notify_progress(f"⏭️ 跳過重複 {category_name} 文章: {article.title[:30]}...")
                             else:
                                 self._notify_progress(f"⏭️ 跳過重複 {category_name} 文章: {article.title[:30]}...")
                     
                     articles.append(article)
+                else:
+                    error_count += 1
                 
                 # 文章間延遲
                 if i < len(article_urls):
-                    time.sleep(random.uniform(5, 10))
+                    time.sleep(random.uniform(3, 8))  # 縮短延遲時間
                 
             except Exception as e:
                 logger.error(f"處理文章時發生錯誤 {url}: {e}")
                 self._notify_progress(f"❌ 處理 {category_name} 文章時發生錯誤: {str(e)[:50]}...")
+                error_count += 1
                 continue
         
-        self._notify_progress(f"✅ {category_name} 分類爬取完成，共處理 {len(articles)} 篇文章")
+        self._notify_progress(f"✅ {category_name} 分類爬取完成，共處理 {len(articles)} 篇文章（成功: {success_count}, 錯誤: {error_count}）")
         return articles
     
-    def crawl_all_categories(self, max_articles_per_category: int = 8) -> Dict[str, List[NewsItem]]:
-        """爬取所有分類的新聞"""
+    def crawl_all_categories(self, max_articles_per_category: int = None) -> Dict[str, List[NewsItem]]:
+        """爬取所有分類的新聞 - 移除限制"""
         results = {}
         
         for category in self.categories.keys():
             try:
                 category_name = "美股" if category == "us_stock" else "台股"
                 self._notify_progress(f"🎯 開始爬取 {category_name} 分類")
+                
+                # **關鍵修正：傳遞None表示不限制**
                 articles = self.crawl_category(category, max_articles_per_category)
                 results[category] = articles
                 
                 # 分類間延遲
                 if len(self.categories) > 1:
                     self._notify_progress(f"⏸️ 分類間休息...")
-                    time.sleep(random.uniform(30, 60))
+                    time.sleep(random.uniform(20, 40))  # 縮短休息時間
                 
             except Exception as e:
                 logger.error(f"爬取 {category} 分類時發生錯誤: {e}")