Update crawler.py

crawler.py

@@ -6,7 +6,7 @@ import random
 import logging
 import re
 from datetime import datetime, timedelta
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Callable
 from urllib.parse import urljoin, urlparse
 from fake_useragent import UserAgent
 import json
@@ -27,9 +27,9 @@ class NewsItem:
     sentiment_score: Optional[float] = None
 
 class CnYesNewsCrawler:
-    """鉅亨網新聞爬蟲 - 改進版"""
+    """鉅亨網新聞爬蟲 - 即時分析版"""
     
-    def __init__(self):
+    def __init__(self, sentiment_analyzer=None, database=None):
         self.base_url = "https://news.cnyes.com"
         self.session = cloudscraper.create_scraper(
             browser={
@@ -40,15 +40,32 @@ class CnYesNewsCrawler:
         )
         self.ua = UserAgent()
         
+        # 注入依賴
+        self.sentiment_analyzer = sentiment_analyzer
+        self.database = database
+        
         # 新聞分類URL
         self.categories = {
             'us_stock': 'https://news.cnyes.com/news/cat/us_stock',
             'tw_stock': 'https://news.cnyes.com/news/cat/tw_stock_news'
         }
         
+        # 進度回調函數
+        self.progress_callback = None
+        
         # 設置請求頭
         self._setup_headers()
         
+    def set_progress_callback(self, callback: Callable[[str], None]):
+        """設置進度回調函數"""
+        self.progress_callback = callback
+        
+    def _notify_progress(self, message: str):
+        """通知進度更新"""
+        if self.progress_callback:
+            self.progress_callback(message)
+        logger.info(message)
+        
     def _setup_headers(self):
         """設置更真實的請求頭"""
         self.session.headers.update({
@@ -70,13 +87,11 @@ class CnYesNewsCrawler:
         })
     
     def _get_page(self, url: str, retries: int = 3) -> Optional[BeautifulSoup]:
-        """獲取網頁內容 - 改進版"""
+        """獲取網頁內容"""
         for attempt in range(retries):
             try:
-                # 更長的隨機延遲，模擬人類行為
-                time.sleep(random.uniform(8, 15))
+                time.sleep(random.uniform(3, 8))
                 
-                # 輪換 User-Agent
                 user_agents = [
                     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
                     'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
@@ -99,12 +114,12 @@ class CnYesNewsCrawler:
             except Exception as e:
                 logger.error(f"請求失敗 (嘗試 {attempt + 1}/{retries}): {e}")
                 if attempt < retries - 1:
-                    time.sleep(random.uniform(15, 30))
+                    time.sleep(random.uniform(5, 15))
                     
         return None
     
     def _extract_article_urls(self, category_url: str, max_pages: int = 2) -> List[str]:
-        """從分類頁面提取文章URL - 改進版"""
+        """從分類頁面提取文章URL"""
         article_urls = []
         
         for page in range(1, max_pages + 1):
@@ -112,16 +127,14 @@ class CnYesNewsCrawler:
                 if page == 1:
                     url = category_url
                 else:
-                    # 修正分頁URL格式
                     url = f"{category_url}?page={page}"
                 
-                logger.info(f"爬取分類頁面 {page}: {url}")
+                self._notify_progress(f"🔍 爬取分類頁面 {page}: {url}")
                 soup = self._get_page(url)
                 
                 if not soup:
                     continue
                 
-                # 改進的選擇器，針對鉅亨網的實際結構
                 link_selectors = [
                     'a[href*="/news/id/"]',
                     '.news-list a[href*="/news/id/"]',
@@ -146,36 +159,31 @@ class CnYesNewsCrawler:
                             page_urls.append(full_url)
                 
                 article_urls.extend(page_urls)
-                logger.info(f"第 {page} 頁找到 {len(page_urls)} 篇文章")
+                self._notify_progress(f"📄 第 {page} 頁找到 {len(page_urls)} 篇文章")
                 
                 if not page_urls:
-                    logger.warning(f"第 {page} 頁沒有找到文章，可能遇到反爬蟲機制")
+                    logger.warning(f"第 {page} 頁沒有找到文章")
                     break
                     
-                # 頁面間更長延遲
                 if page < max_pages:
-                    time.sleep(random.uniform(20, 40))
+                    time.sleep(random.uniform(8, 15))
                     
             except Exception as e:
                 logger.error(f"爬取第 {page} 頁時發生錯誤: {e}")
                 continue
         
-        # 去重並限制數量
         unique_urls = list(set(article_urls))
-        logger.info(f"總共找到 {len(unique_urls)} 篇獨特文章")
+        self._notify_progress(f"🎯 總共找到 {len(unique_urls)} 篇獨特文章")
         return unique_urls
     
     def _extract_article_content(self, url: str, category: str) -> Optional[NewsItem]:
-        """提取文章詳細內容 - 改進版"""
+        """提取文章詳細內容"""
         try:
             soup = self._get_page(url)
             if not soup:
                 return None
             
-            # 調試：打印網頁結構的一部分
-            logger.info(f"網頁標題標籤: {[tag.name for tag in soup.find_all(['h1', 'h2', 'h3'])]}")
-            
-            # 改進的標題選擇器
+            # 提取標題
             title_selectors = [
                 'h1[class*="title"]',
                 'h1.news-title',
@@ -195,22 +203,18 @@ class CnYesNewsCrawler:
                 if title_elem:
                     title = title_elem.get_text(strip=True)
                     if title and len(title) > 10:
-                        logger.info(f"使用選擇器 '{selector}' 找到標題: {title[:50]}...")
                         break
             
             if not title:
-                logger.warning(f"無法提取標題: {url}")
-                # 嘗試從頁面標題獲取
                 page_title = soup.find('title')
                 if page_title:
                     title = page_title.get_text(strip=True).split(' | ')[0]
-                    logger.info(f"從頁面標題獲取: {title[:50]}...")
             
             if not title or len(title) < 5:
                 logger.warning(f"標題太短或無法提取: {url}")
                 return None
             
-            # 改進的內容選擇器
+            # 提取內容
             content_selectors = [
                 '.article-content',
                 '.news-content', 
@@ -228,11 +232,9 @@ class CnYesNewsCrawler:
             for selector in content_selectors:
                 content_container = soup.select_one(selector)
                 if content_container:
-                    # 移除不需要的元素
                     for unwanted in content_container.select('script, style, .ad, .advertisement, .related, .share, .comment'):
                         unwanted.decompose()
                     
-                    # 提取文本段落
                     paragraphs = content_container.find_all(['p', 'div'], string=True)
                     content_parts = []
                     
@@ -243,33 +245,10 @@ class CnYesNewsCrawler:
                     
                     content = '\n'.join(content_parts)
                     if len(content) > 100:
-                        logger.info(f"使用選擇器 '{selector}' 找到內容，長度: {len(content)}")
                         break
             
-            # 如果還是沒有內容，嘗試獲取所有文本
-            if not content or len(content) < 100:
-                logger.warning(f"常規方法無法提取內容，嘗試備用方法: {url}")
-                
-                # 移除不需要的標籤
-                for unwanted in soup.select('script, style, nav, header, footer, .menu, .sidebar, .ad'):
-                    unwanted.decompose()
-                
-                # 尋找包含最多文本的元素
-                all_text_elements = soup.find_all(['p', 'div'], string=True)
-                text_blocks = []
-                
-                for elem in all_text_elements:
-                    text = elem.get_text(strip=True)
-                    if len(text) > 50:
-                        text_blocks.append(text)
-                
-                if text_blocks:
-                    content = '\n'.join(text_blocks[:10])  # 取前10段
-                    logger.info(f"備用方法找到內容，長度: {len(content)}")
-            
             if not content or len(content) < 50:
-                logger.warning(f"內容太短或無法提取: {url}, 內容長度: {len(content)}")
-                logger.debug(f"網頁HTML結構預覽: {str(soup)[:500]}...")
+                logger.warning(f"內容太短或無法提取: {url}")
                 return None
             
             # 提取發布時間
@@ -281,14 +260,13 @@ class CnYesNewsCrawler:
             # 創建新聞項目
             news_item = NewsItem(
                 title=title,
-                content=content[:2000],  # 限制內容長度
+                content=content[:2000],
                 url=url,
                 source='鉅亨網',
                 category=category,
                 published_date=published_date
             )
             
-            logger.info(f"成功提取文章: {title[:50]}... (內容長度: {len(content)})")
             return news_item
             
         except Exception as e:
@@ -297,13 +275,9 @@ class CnYesNewsCrawler:
     
     def _clean_content(self, content: str) -> str:
         """清理內容"""
-        # 移除多餘空白
         content = re.sub(r'\s+', ' ', content)
-        
-        # 移除特殊字符
         content = re.sub(r'[^\u4e00-\u9fff\u3400-\u4dbf\w\s.,!?()（），。！？：；「」『』]', '', content)
         
-        # 移除重複句子
         sentences = content.split('。')
         unique_sentences = []
         for sentence in sentences:
@@ -313,7 +287,7 @@ class CnYesNewsCrawler:
         return '。'.join(unique_sentences)
     
     def _extract_publish_date(self, soup: BeautifulSoup) -> datetime:
-        """提取發布時間 - 改進版"""
+        """提取發布時間"""
         time_selectors = [
             'time[datetime]',
             '.publish-time',
@@ -328,7 +302,6 @@ class CnYesNewsCrawler:
         for selector in time_selectors:
             time_elem = soup.select_one(selector)
             if time_elem:
-                # 檢查datetime屬性
                 datetime_attr = time_elem.get('datetime') or time_elem.get('content')
                 if datetime_attr:
                     try:
@@ -336,7 +309,6 @@ class CnYesNewsCrawler:
                     except:
                         pass
                 
-                # 檢查文本內容
                 time_text = time_elem.get_text(strip=True)
                 parsed_time = self._parse_time_text(time_text)
                 if parsed_time:
@@ -345,7 +317,7 @@ class CnYesNewsCrawler:
         return datetime.now()
     
     def _parse_time_text(self, time_text: str) -> Optional[datetime]:
-        """解析時間文字 - 改進版"""
+        """解析時間文字"""
         patterns = [
             r'(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2}):(\d{2})',
             r'(\d{4})-(\d{2})-(\d{2})\s+(\d{2}):(\d{2})',
@@ -374,62 +346,101 @@ class CnYesNewsCrawler:
         return None
     
     def crawl_category(self, category: str, max_articles: int = 10) -> List[NewsItem]:
-        """爬取指定分類的新聞 - 減少數量避免被封"""
+        """爬取指定分類的新聞 - 即時分析版"""
         if category not in self.categories:
             logger.error(f"無效的分類: {category}")
             return []
         
-        logger.info(f"開始爬取 {category} 分類新聞")
+        self._notify_progress(f"🚀 開始爬取 {category} 分類新聞")
         
         # 獲取文章URL列表
         category_url = self.categories[category]
         article_urls = self._extract_article_urls(category_url, max_pages=2)
         
         if not article_urls:
-            logger.warning(f"未找到 {category} 分類的文章URL")
+            self._notify_progress(f"⚠️ 未找到 {category} 分類的文章URL")
             return []
         
-        # 限制文章數量，避免被封
+        # 限制文章數量
         if len(article_urls) > max_articles:
             article_urls = article_urls[:max_articles]
         
-        # 提取文章內容
+        # 提取文章內容並即時分析存檔
         articles = []
         for i, url in enumerate(article_urls, 1):
             try:
-                logger.info(f"處理文章 {i}/{len(article_urls)}: {url}")
+                self._notify_progress(f"📖 處理文章 {i}/{len(article_urls)}: 正在提取內容...")
                 article = self._extract_article_content(url, category)
+                
                 if article:
+                    # 即時情感分析
+                    if self.sentiment_analyzer:
+                        self._notify_progress(f"🧠 分析文章 {i}/{len(article_urls)}: {article.title[:30]}...")
+                        sentiment_result = self.sentiment_analyzer.analyze_sentiment(
+                            article.content, article.title
+                        )
+                        article.sentiment = sentiment_result['sentiment']
+                        article.sentiment_score = sentiment_result['confidence']
+                        
+                        # 即時存檔
+                        if self.database:
+                            # 檢查重複
+                            if not self.database.check_duplicate_by_title(article.title):
+                                db_article = {
+                                    'title': article.title,
+                                    'content': article.content,
+                                    'url': article.url,
+                                    'source': article.source,
+                                    'category': article.category,
+                                    'published_date': article.published_date.isoformat(),
+                                    'sentiment': article.sentiment,
+                                    'sentiment_score': article.sentiment_score,
+                                    'sentiment_method': 'auto'
+                                }
+                                
+                                inserted, _ = self.database.insert_news([db_article])
+                                if inserted > 0:
+                                    self._notify_progress(f"💾 已保存文章: {article.title[:30]}... (情緒: {article.sentiment})")
+                                else:
+                                    self._notify_progress(f"⏭️ 跳過重複文章: {article.title[:30]}...")
+                            else:
+                                self._notify_progress(f"⏭️ 跳過重複文章: {article.title[:30]}...")
+                    
                     articles.append(article)
                 
-                # 更長的隨機延遲，模擬人類閱讀
-                time.sleep(random.uniform(15, 30))
+                # 文章間延遲
+                if i < len(article_urls):
+                    time.sleep(random.uniform(5, 10))
                 
             except Exception as e:
                 logger.error(f"處理文章時發生錯誤 {url}: {e}")
+                self._notify_progress(f"❌ 處理文章時發生錯誤: {str(e)[:50]}...")
                 continue
         
-        logger.info(f"{category} 分類爬取完成，共 {len(articles)} 篇文章")
+        self._notify_progress(f"✅ {category} 分類爬取完成，共處理 {len(articles)} 篇文章")
         return articles
     
     def crawl_all_categories(self, max_articles_per_category: int = 8) -> Dict[str, List[NewsItem]]:
-        """爬取所有分類的新聞 - 減少數量"""
+        """爬取所有分類的新聞 - 即時分析版"""
         results = {}
         
         for category in self.categories.keys():
             try:
-                logger.info(f"開始爬取 {category} 分類")
+                self._notify_progress(f"🎯 開始爬取 {category} 分類")
                 articles = self.crawl_category(category, max_articles_per_category)
                 results[category] = articles
                 
-                # 分類間更長延遲
-                time.sleep(random.uniform(60, 120))
+                # 分類間延遲
+                if len(self.categories) > 1:
+                    self._notify_progress(f"⏸️ 分類間休息...")
+                    time.sleep(random.uniform(30, 60))
                 
             except Exception as e:
                 logger.error(f"爬取 {category} 分類時發生錯誤: {e}")
+                self._notify_progress(f"❌ 爬取 {category} 分類時發生錯誤: {str(e)}")
                 results[category] = []
         
         total_articles = sum(len(articles) for articles in results.values())
-        logger.info(f"所有分類爬取完成，總共 {total_articles} 篇文章")
+        self._notify_progress(f"🎉 所有分類爬取完成，總共處理 {total_articles} 篇文章")
         
         return results