Upload 2 files

app.py

@@ -4,8 +4,9 @@ import random
 import re
 import time
 import xml.etree.ElementTree as ET
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Tuple
 from concurrent.futures import ThreadPoolExecutor, as_completed
+from urllib.parse import urlparse
 
 import requests
 from fastapi import FastAPI
@@ -540,18 +541,24 @@ def fetch_html_articles(source: Dict) -> List[Dict]:
 
 
 def fetch_articles_from_source(source: Dict) -> List[Dict]:
-    """統一入口：根據來源類型呼叫對應的擷取函式。"""
+    """統一入口：根據來源類型呼叫對應的擷取函式；失敗時自動切換 HTML 備用爬蟲。"""
+    articles = []
     try:
         if source["type"] == "rss":
-            return fetch_rss_articles(source)
+            articles = fetch_rss_articles(source)
         elif source["type"] == "html":
-            return fetch_html_articles(source)
+            articles = fetch_html_articles(source)
         else:
             print(f"⚠️ 未知來源類型: {source['type']}", flush=True)
-            return []
     except Exception as e:
         print(f"❌ [{source['name']}] 擷取異常: {e}", flush=True)
-        return []
+
+    # ── 若正常抓取失敗（0 篇），自動啟動 HTML 備用爬蟲 ──
+    if not articles:
+        print(f"🔄 [{source['name']}] 正常抓取取得 0 篇，觸發 HTML 備用爬蟲...", flush=True)
+        articles = fetch_html_fallback(source)
+
+    return articles
 
 
 # ─────────────────────────────────────────────────────────────────
@@ -610,6 +617,146 @@ SYSTEM_PROMPT = """你是一個專業的繁體中文新聞編輯秘書。你將
 - 回傳的 JSON 陣列長度必須等於輸入的文章數量"""
 
 
+# ── HTML 備用爬蟲專用提示詞 ──
+HTML_FALLBACK_SYSTEM_PROMPT = """你是一個精準的網頁新聞提煉專家。我將提供一段從網站首頁擷取下來的 HTML 純文字內容（已移除 script/style 標籤）。
+這段文字混雜了選單、廣告、頁尾與真正的新聞條目。
+
+你的任務是：
+1. 從這段雜亂的文字中，精確找出**最新的 2 篇重要新聞**
+2. 自行判斷原文語言，遵循中英雙語對照規則（與主新聞秘書相同）：
+   - 外語原文 → 上半段繁體中文 / 下半段英文對照
+   - 中文原文 → 僅繁體中文
+3. 繁體中文摘要 80～200 字，英文摘要 40～70 字
+4. 為每篇新聞指定一個 category（從：科技 | 財經 | 國際 | 旅遊 | 生活 | 科學 | 材料工業 | 冷知識 | 熱門趨勢 中挑選）
+5. 圖片欄位（image_url）若無法從文本中取得，請留空字串 ""
+
+你必須**嚴格回傳一個 JSON 物件**，內含 "articles" 陣列：
+{
+  "articles": [
+    {
+      "title": "...",
+      "summary": "...",
+      "category": "...",
+      "image_url": "",
+      "source": "提供的網站名稱"
+    },
+    ...
+  ]
+}
+
+注意：只回傳真正的新聞內容，忽略導覽選單、頁尾連結、廣告、社交媒體按鈕等雜訊。"""
+
+
+def fetch_html_fallback(source: Dict) -> List[Dict]:
+    """
+    HTML 備用爬蟲：
+    當 RSS/正常爬蟲失敗時，抓取網站首頁並請 DeepSeek 直接從雜亂文本中提取新聞。
+    回傳的文章已具備最終格式（含 title, summary, category, image_url, source）。
+    """
+    name = source["name"]
+    url = source["url"]
+
+    # 從 URL 提取首頁網址
+    try:
+        parsed = urlparse(url)
+        base_url = f"{parsed.scheme}://{parsed.netloc}"
+    except Exception:
+        base_url = url
+
+    print(f"🔄 [{name}] RSS 失敗，啟動 HTML 備用爬蟲 → {base_url}", flush=True)
+
+    try:
+        resp = requests.get(base_url, headers=HEADERS, timeout=MAX_FETCH_SECONDS)
+        resp.raise_for_status()
+        resp.encoding = resp.apparent_encoding or "utf-8"
+    except Exception as e:
+        print(f"  ❌ [{name}] HTML 備用連線失敗: {e}", flush=True)
+        return []
+
+    # ── 提取 body 純文字，限制 10000 字元 ──
+    try:
+        soup = BeautifulSoup(resp.text, "html.parser")
+        # 移除 script / style / nav / footer
+        for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
+            tag.decompose()
+        if soup.body:
+            text = soup.body.get_text(separator="\n", strip=True)
+        else:
+            text = soup.get_text(separator="\n", strip=True)
+        # 壓縮空白
+        text = re.sub(r"\n\s*\n", "\n", text)
+        text = re.sub(r" +", " ", text)
+        text = text[:10000]
+        print(f"  📄 [{name}] 提取純文字 {len(text)} 字元，餵給 DeepSeek...", flush=True)
+    except Exception as e:
+        print(f"  ❌ [{name}] HTML 文本提取失敗: {e}", flush=True)
+        return []
+
+    if len(text) < 100:
+        print(f"  ⚠️ [{name}] HTML 文本過短，跳過", flush=True)
+        return []
+
+    # ── 餵給 DeepSeek 提取新聞 ��─
+    try:
+        response = client.chat.completions.create(
+            model=DEEPSEEK_MODEL,
+            messages=[
+                {"role": "system", "content": HTML_FALLBACK_SYSTEM_PROMPT},
+                {"role": "user", "content": f"網站名稱：{name}\n網址：{base_url}\n\nHTML 文本內容：\n{text}"},
+            ],
+            response_format={"type": "json_object"},
+            temperature=0.3,
+            max_tokens=4096,
+        )
+
+        raw = response.choices[0].message.content
+        result = json.loads(raw)
+
+        # 提取 articles 陣列
+        articles = []
+        if isinstance(result, dict):
+            for key in ["articles", "news", "items", "results", "data"]:
+                if key in result and isinstance(result[key], list):
+                    articles = result[key]
+                    break
+            # 若只有單個 key 是 list
+            if not articles:
+                for val in result.values():
+                    if isinstance(val, list):
+                        articles = val
+                        break
+        elif isinstance(result, list):
+            articles = result
+
+        # ── 補完欄位 ──
+        completed = []
+        for item in articles[:5]:  # 最多取 5 篇
+            if not isinstance(item, dict):
+                continue
+            completed.append({
+                "title": str(item.get("title", "未知標題")).strip(),
+                "summary": str(item.get("summary", item.get("content", "暫無摘要"))).strip(),
+                "category": str(item.get("category", guess_category(name))).strip(),
+                "image_url": item.get("image_url") or get_fallback_image(str(item.get("title", ""))),
+                "source": name,
+                "_fallback": True,  # 標記為備用爬蟲產出（已含摘要，不需再次 summarize）
+            })
+
+        if completed:
+            print(f"  ✅ [{name}] HTML 備用爬蟲成功提取 {len(completed)} 篇新聞", flush=True)
+        else:
+            print(f"  ⚠️ [{name}] HTML 備用爬蟲未找到新聞", flush=True)
+
+        return completed
+
+    except json.JSONDecodeError as e:
+        print(f"  ❌ [{name}] DeepSeek JSON 解析失敗: {e}", flush=True)
+    except Exception as e:
+        print(f"  ❌ [{name}] HTML 備用爬蟲 DeepSeek 呼叫失敗: {e}", flush=True)
+
+    return []
+
+
 def summarize_by_deepseek(articles: List[Dict]) -> List[Dict]:
     """將文章餵給 DeepSeek 進行摘要處理，回傳結構化 JSON 陣列。"""
     if not articles:
@@ -748,10 +895,11 @@ def home():
 def get_news():
     """
     主新聞端點：
-    1. 從 5 個來源各抓 15 篇最新新聞
-    2. 確保每篇都有圖片
-    3. 餵給 DeepSeek 做繁體中文摘要
-    4. 回傳統一 JSON 陣列
+    1. 從 news_sources.json 載入的所有來源平行抓取最新新聞
+    2. RSS/HTML 失敗時自動切換 HTML 備用爬蟲（DeepSeek 直接提取）
+    3. 確保每篇都有圖片
+    4. 一般文章餵給 DeepSeek 做繁體中文摘要（備用文章已含摘要則略過）
+    5. 回傳統一 JSON 陣列
     """
     start_time = time.time()
     all_articles: List[Dict] = []
@@ -783,22 +931,35 @@ def get_news():
             "articles": [],
         }
 
-    # ── 步驟 2：餵給 DeepSeek 摘要 ──
-    # 若文章數量過多，分批次處理（每批最多 30 篇）
+    # ── 步驟 2：分離「備用爬蟲文章」（已含 DeepSeek 摘要）與「一般文章」──
+    regular_articles: List[Dict] = []
+    fallback_articles: List[Dict] = []
+
+    for a in all_articles:
+        if a.pop("_fallback", False):
+            fallback_articles.append(a)
+        else:
+            regular_articles.append(a)
+
+    fb_count = len(fallback_articles)
+    reg_count = len(regular_articles)
+    print(f"📊 一般文章 {reg_count} 篇 + 備用爬蟲文章 {fb_count} 篇（已摘要，略過處理）", flush=True)
+
+    # ── 步驟 3：一般文章餵給 DeepSeek 摘要 ──
     BATCH_SIZE = 30
-    final_articles: List[Dict] = []
+    final_articles: List[Dict] = list(fallback_articles)  # 備用文章直接加入
 
-    if len(all_articles) <= BATCH_SIZE:
-        final_articles = summarize_by_deepseek(all_articles)
-    else:
-        for i in range(0, len(all_articles), BATCH_SIZE):
-            batch = all_articles[i:i + BATCH_SIZE]
-            print(f"📦 處理批次 {i // BATCH_SIZE + 1}/{(len(all_articles) + BATCH_SIZE - 1) // BATCH_SIZE} ({len(batch)} 篇)...", flush=True)
-            batch_result = summarize_by_deepseek(batch)
-            final_articles.extend(batch_result)
+    if regular_articles:
+        if len(regular_articles) <= BATCH_SIZE:
+            final_articles.extend(summarize_by_deepseek(regular_articles))
+        else:
+            for i in range(0, len(regular_articles), BATCH_SIZE):
+                batch = regular_articles[i:i + BATCH_SIZE]
+                print(f"📦 處理批次 {i // BATCH_SIZE + 1}/{(len(regular_articles) + BATCH_SIZE - 1) // BATCH_SIZE} ({len(batch)} 篇)...", flush=True)
+                final_articles.extend(summarize_by_deepseek(batch))
 
     total_time = time.time() - start_time
-    print(f"🏁 全部完成：{len(final_articles)} 篇新聞（總耗時 {total_time:.1f}s）", flush=True)
+    print(f"🏁 全部完成：{len(final_articles)} 篇新聞（一般 {reg_count} + 備用 {fb_count}）（總耗時 {total_time:.1f}s）", flush=True)
     print(f"{'='*60}\n", flush=True)
 
     return final_articles

news_sources.json

@@ -1,29 +1,13 @@
 [
   {"name": "科技新報",         "type": "rss",  "url": "https://technews.tw/feed/",                              "category": "科技/AI"},
-  {"name": "癮科技",           "type": "rss",  "url": "https://chinese.engadget.com/rss.xml",                    "category": "科技/3C"},
-  {"name": "數位時代",         "type": "rss",  "url": "https://www.bnext.com/rss",                               "category": "科技/商業"},
+  {"name": "數位時代",         "type": "rss",  "url": "https://www.bnext.com/",                               "category": "科技/商業"},
   {"name": "HKEPC",            "type": "html", "url": "https://www.hkepc.com/",                                  "category": "科技/硬體"},
-  {"name": "QbitAI 機器之心",  "type": "rss",  "url": "https://www.jiqizhixin.com/rss",                          "category": "科技/AI"},
-  {"name": "Inside 科技趨勢",  "type": "rss",  "url": "https://www.inside.com.tw/feed",                          "category": "科技/趨勢"},
+  {"name": "QbitAI 機器之心",  "type": "rss",  "url": "https://www.jiqizhixin.com/",                          "category": "科技/AI"},
+  {"name": "Inside 科技趨勢",  "type": "rss",  "url": "https://www.inside.com.tw/",                          "category": "科技/趨勢"},
   {"name": "明日科學",         "type": "rss",  "url": "https://tomorrowsci.com/feed/",                           "category": "科學/未來"},
   {"name": "Gizmodo",          "type": "rss",  "url": "https://gizmodo.com/rss",                                 "category": "科技/極客"},
   {"name": "Hackaday",         "type": "rss",  "url": "https://hackaday.com/feed/",                              "category": "科技/DIY"},
-  {"name": "地球圖輯隊",       "type": "rss",  "url": "https://world.yam.com/rss.php",                           "category": "國際/圖文"},
-  {"name": "冷知識",           "type": "rss",  "url": "https://misstwocm.com/feed/",                             "category": "生活/趣味"},
   {"name": "鉅亨網 國際政經",   "type": "rss",  "url": "https://news.cnyes.com/rss/div/global_macro",            "category": "財經/國際"},
-  {"name": "阿斯達克財經網",    "type": "rss",  "url": "https://www.aastocks.com/tc/resources/rss.ashx?type=1",  "category": "財經/港股"},
-  {"name": "RTHK 財經新聞",    "type": "rss",  "url": "https://rthk.hk/rthk/news/rss/c_expressnews_cfinance.xml",       "category": "財經/香港"},
-  {"name": "RTHK 本地新聞",    "type": "rss",  "url": "https://rthk.hk/rthk/news/rss/c_expressnews_clocal.xml",          "category": "時事/香港"},
-  {"name": "RTHK 國際新聞",    "type": "rss",  "url": "https://rthk.hk/rthk/news/rss/c_expressnews_cinternational.xml",  "category": "國際/時事"},
-  {"name": "Plastics Today",   "type": "rss",  "url": "https://www.plasticstoday.com/rss.xml",                    "category": "材料工業/塑膠"},
-  {"name": "Metal Miner",      "type": "rss",  "url": "https://agmetalminer.com/feed/",                           "category": "材料工業/金屬"},
-  {"name": "AZoM 材料科學",    "type": "rss",  "url": "https://www.azom.com/azom-news-feed.xml",                  "category": "材料工業/科學"},
-  {"name": "Wave 流行潮流",     "type": "rss",  "url": "https://www.wavetv.tw/feed/",                             "category": "生活/潮流"},
-  {"name": "U Travel 旅遊",     "type": "rss",  "url": "https://utravel.com.hk/rss",                              "category": "旅遊/香港"},
-  {"name": "Yahoo 旅遊 港台",   "type": "rss",  "url": "https://travel.yahoo.com.tw/rss/headline/",               "category": "旅遊/港台"},
-  {"name": "Yahoo 旅遊 香港",   "type": "rss",  "url": "https://hk.news.yahoo.com/rss/travel",                    "category": "旅遊/香港"},
-  {"name": "港生活 北上",       "type": "rss",  "url": "https://hk.ulifestyle.com.hk/rss/travel-main",           "category": "旅遊/深圳"},
-  {"name": "香港01 大灣區",     "type": "rss",  "url": "https://www.hk01.com/rss/category/877",                   "category": "旅遊/大灣區"},
   {"name": "Google Trends 香港","type": "rss",  "url": "https://trends.google.com.hk/trending/rss?geo=HK",        "category": "熱門趨勢/香港"},
   {"name": "Google Trends 美國","type": "rss",  "url": "https://trends.google.com/trending/rss?geo=US",           "category": "熱門趨勢/全球"}
 ]