Upload 2 files

app.py

@@ -0,0 +1,576 @@
+# -*- coding: utf-8 -*-
+# 財政部財政資訊中心 江信宗
+
+import gradio as gr
+from openai import OpenAI
+import requests
+from bs4 import BeautifulSoup
+import resend
+import random
+import lxml
+import os
+import time
+import pytz
+import datetime
+
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+    'sec-ch-ua': '"Chromium";v="130", "Google Chrome";v="130", "Not?A_Brand";v="99"',
+    'sec-ch-ua-mobile': '?0',
+    'sec-ch-ua-platform': '"Windows"',
+    'sec-fetch-dest': 'document',
+    'sec-fetch-mode': 'navigate',
+    'sec-fetch-site': 'none',
+    'Sec-Fetch-User': '?1',
+    'sec-gpc': '1',
+    'Cache-Control': 'no-cache',
+    'Pragma': 'no-cache',
+    'dnt': '1',
+    'accept-language': 'zh-TW'
+}
+
+custom_css = """
+.center-aligned {
+    text-align: center;
+}
+.text-background {
+    font-size: 20px !important;
+    padding: 5px !important;
+    border-radius: 10px !important;
+    border: 2px solid #B7E0FF !important;
+    margin: 0 !important;
+}
+.file-background {
+    background-color: #B7E0FF !important;
+    padding: 15px !important;
+    border-radius: 10px !important;
+    margin: 0 !important;
+    height: auto;
+}
+.script-background {
+    background-color: #FEF9D9 !important;
+    padding: 5px !important;
+    border-radius: 10px !important;
+    margin: 0 !important;
+}
+#submit-btn {
+    border-radius: 10px !important;
+    border: none !important;
+    background-color: #ff4081 !important;
+    color: white !important;
+    font-weight: bold !important;
+    transition: all 0.3s ease !important;
+    margin: 0 !important;
+}
+#submit-btn:hover {
+    background-color: #f50057 !important;
+    transform: scale(1.05);
+}
+"""
+
+def CheckConnect(url, ref):
+    global headers
+    RefererList = {'UdnSec':'https://udn.com/search/', 'CNA':'https://www.cna.com.tw/', 'LTN':'https://news.ltn.com.tw/list/breakingnews/'}
+    headers['referer'] = RefererList[ref]
+    try:
+        response = requests.get(url, headers=headers)
+        soup = BeautifulSoup(response.text, 'lxml')
+        checkSuccess = True
+        return soup, checkSuccess
+    except Exception as e:
+        print(f'※【爬取失敗】{url} ※')
+        soup = None
+        checkSuccess = False
+        return soup, checkSuccess
+
+def is_peak_hours():
+    taiwan_tz = pytz.timezone('Asia/Taipei')
+    current_time = datetime.datetime.now(taiwan_tz).time()
+    peak_start = datetime.time(8, 0)
+    peak_end = datetime.time(20, 0)
+    return peak_start <= current_time <= peak_end
+
+def UdnTax_Breaknews(num_news):
+    url = 'https://udn.com/api/more'
+    news_list = []
+    for page in range(1, 3):
+        params = {
+            'page': page,
+            'channelId': 1,
+            'type': 'breaknews',
+            'cate_id': 0,
+            'totalRecNo': 10655
+        }
+        try:
+            response = requests.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            data = response.json()
+            if data['state']:
+                for article in data['lists']:
+                    try:
+                        Title = article['title']
+                        if any(substring in Title for substring in ["中職", "棒球", "MLB", "大選", "新聞周刊", "TPBL", "即時短評", "醫奉", "週運勢", "統一獅", "中信兄弟"]):
+                            continue
+                        Link = 'https://udn.com' + article['titleLink']
+                        Link = Link.split('?')[0].strip()
+                        AMP_Link = r'/news/amp/' + Link.split(r'news/')[1].strip()
+                        AMP_Link = 'https://udn.com' + AMP_Link
+                        checkSuccess = False
+                        tryNums = 0
+                        while not checkSuccess and tryNums < 2:
+                            soup, checkSuccess = CheckConnect(AMP_Link, 'UdnSec')
+                            if not checkSuccess:
+                                tryNums += 1
+                                time.sleep(2)
+                        if not checkSuccess:
+                            print(f'開啟連結累積失敗2次，跳過此文章：{Title}')
+                            continue
+                        if soup is None:
+                            print(f'無法解析文章內容，跳過此文章：{Title}')
+                            continue
+                        Summary = soup.select_one('main.main')
+                        if Summary is None:
+                            print(f'無法找到文章主體，跳過此文章：{Title}')
+                            continue
+                        Section = Summary.select('p')
+                        Article = ''
+                        for context in Section:
+                            context = context.get_text().strip()
+                            context = context.replace('\n','')
+                            Article = Article + context
+                        Article = Article.split('《原文刊登')[0].strip()
+                        news_list.append((Title, Link, Article))
+                        if len(news_list) >= int(num_news):
+                            return news_list
+                    except Exception as e:
+                        print(f"Error processing article in UdnTax_Breaknews: {e}")
+            else:
+                print(f"UdnTax_Breaknews: API response state is false for page {page}")
+        except requests.RequestException as e:
+            print(f"Error in UdnTax_Breaknews request for page {page}: {e}")
+        except ValueError as e:
+            print(f"Error parsing JSON in UdnTax_Breaknews for page {page}: {e}")
+        except Exception as e:
+            print(f"Unexpected error in UdnTax_Breaknews for page {page}: {e}")
+    return news_list
+
+def CNA_Breaknews(num_news):
+    url = 'https://www.cna.com.tw/list/aall.aspx'
+    soup, checkSuccess = CheckConnect(url, 'CNA')
+    if not checkSuccess:
+        return []
+    news_list = []
+    try:
+        summary = soup.select_one('div.centralContent')
+        if summary:
+            section = summary.select('li')
+            for context in section:
+                try:
+                    title = context.select_one('h2').get_text().strip()
+                    if any(title.startswith(prefix) for prefix in [
+                        "大樂透",
+                        "今彩539",
+                        "威力彩",
+                        "一週大事",
+                        "開箱老照片》"
+                    ]):
+                        continue
+                    link = context.select_one('a')['href'].strip()
+                    if any(substring in link for substring in ["/aspt/", "/amov/", "/ahel/", "netzero."]):
+                        continue
+                    link = 'https://www.cna.com.tw' + link
+                    article_soup, _ = CheckConnect(link, 'CNA')
+                    if article_soup:
+                        article = article_soup.select_one('div.paragraph')
+                        if article:
+                            article = article.get_text().strip()
+                            article = article.replace('\n','')
+                            article = article.split('（編輯：')[0].split('（譯者：')[0]
+                        else:
+                            article = "無法取得文章內容"
+                    else:
+                        article = "無法取得文章內容"
+                    news_list.append((title, link, article))
+                    if len(news_list) >= int(num_news):
+                        break
+                except Exception as e:
+                    print(f"Error processing article in CNA_Breaknews: {e}")
+    except Exception as e:
+        print(f"Error in CNA_Breaknews: {e}")
+    return news_list
+
+def LTN_Breaknews(num_news):
+    url = 'https://news.ltn.com.tw/rss/all.xml'
+    news_list = []
+    try:
+        response = requests.get(url, headers=headers)
+        soup = BeautifulSoup(response.text, 'lxml-xml')
+        items = soup.find_all('item')
+        for item in items:
+            try:
+                link = item.find('link').text.strip()
+                if any(link.startswith(prefix) for prefix in [
+                    "https://health.ltn.com.tw/",
+                    "https://news.ltn.com.tw/news/def/",
+                    "https://def.ltn.com.tw/",
+                    "https://sports.ltn.com.tw/",
+                    "https://istyle.ltn.com.tw/",
+                    "https://ent.ltn.com.tw/",
+                    "https://talk.ltn.com.tw/"
+                ]):
+                    continue
+                title = item.find('title').text.strip()
+                if any(title.startswith(prefix) for prefix in [
+                        "自由爆新聞》",
+                        "自由說新聞》"
+                ]):
+                    continue
+                if any(substring in title for substring in ["各報重點新聞一覽", "大樂透", "威力彩", "今彩539", "統一獅", "中信兄弟", "TPBL"]):
+                    continue
+                article = item.find('description').text.strip()
+                news_list.append((title, link, article))
+                if len(news_list) >= int(num_news):
+                    break
+            except Exception as e:
+                print(f"Error processing article in LTN_Breaknews: {e}")
+    except Exception as e:
+        print(f"Error in LTN_Breaknews: {e}")
+    return news_list
+
+def ETtoday_Breaknews(num_news):
+    url = 'https://feeds.feedburner.com/ettoday/realtime'
+    news_list = []
+    try:
+        response = requests.get(url, headers=headers)
+        soup = BeautifulSoup(response.content, 'xml')
+        items = soup.find_all('item')
+        for item in items:
+            title = item.find('title').text.strip()
+            link = item.find('link').text.strip()
+            if any(link.startswith(prefix) for prefix in [
+                    "https://star.ettoday.net/",
+                    "https://fashion.ettoday.net/",
+                    "https://ezbuy.ettoday.net/",
+                    "https://boba.ettoday.net/",
+                    "https://events.ettoday.net/",
+                    "https://forum.ettoday.net/",
+                    "https://sports.ettoday.net/",
+                    "https://pets.ettoday.net/",
+                    "https://speed.ettoday.net/",
+                    "https://travel.ettoday.net/",
+                    "https://game.ettoday.net/",
+                    "https://health.ettoday.net/",
+                    "https://discovery.ettoday.net/",
+                    "https://esg.ettoday.net/",
+                    "https://ettoday.kaik.io/"
+                ]):
+                continue
+            description = item.find('description').text.strip()
+            content_start = description.find('<br/>')
+            if content_start != -1:
+                content_start += 5
+                article = description[content_start:].strip()
+                article = BeautifulSoup(article, 'html.parser').get_text()
+                if not article:
+                    continue
+            else:
+                continue
+            news_list.append((title, link, article))
+            if len(news_list) >= int(num_news):
+                break
+    except Exception as e:
+        print(f"Error in ETtoday_Breaknews: {e}")
+    return news_list
+
+def SETN_Breaknews(num_news):
+    url = 'https://www.setn.com/ViewAll.aspx'
+    params = {
+        'p': 1
+    }
+    news_list = []
+    try:
+        response = requests.get(url, params=params, headers=headers)
+        soup = BeautifulSoup(response.text, 'html.parser')
+        news_items = soup.select('div.newsItems')
+        for item in news_items:
+            try:
+                title_elem = item.select_one('h3.view-li-title a')
+                if title_elem:
+                    title = title_elem.text.strip()
+                    if any(substring in title for substring in ["中職", "棒球", "MLB", "大選", "會員日", "TPBL", "統一獅", "中信兄弟"]):
+                        continue
+                    link = title_elem['href']
+                    if not link.startswith('http'):
+                        if link.startswith('//'):
+                            link = 'https:' + link
+                        elif link.startswith('/'):
+                            link = 'https://www.setn.com' + link
+                        else:
+                            link = 'https://www.setn.com/' + link
+                    if any(link.startswith(prefix) for prefix in [
+                        "https://star.setn.com",
+                        "https://health.setn.com"
+                    ]):
+                        continue
+                    link = link.split('&utm_')[0]
+                    article_response = requests.get(link, headers=headers)
+                    article_soup = BeautifulSoup(article_response.text, 'html.parser')
+                    article_content = article_soup.select_one('div#Content1')
+                    if article_content:
+                        article = article_content.get_text(strip=True)
+                    else:
+                        article = "無法獲取文章內容"
+                    news_list.append((title, link, article))
+                    if len(news_list) >= int(num_news):
+                        break
+            except Exception as e:
+                print(f"Error processing news item in SETN_Breaknews: {e}")
+    except requests.RequestException as e:
+        print(f"Error in SETN_Breaknews request: {e}")
+    except Exception as e:
+        print(f"Unexpected error in SETN_Breaknews: {e}")
+    return news_list
+
+def initial_news(num_news):
+    gr.Info("掌握焦點新聞中，請稍待片刻....")
+    start_time = time.time()
+    taiwan_tz = pytz.timezone('Asia/Taipei')
+    current_time = datetime.datetime.now(taiwan_tz).strftime("%Y-%m-%d %H:%M:%S")
+    all_news = []
+    total_news = 0
+    news_sources = [
+        ('UdnTax_Breaknews', UdnTax_Breaknews, int(num_news)),
+        ('CNA_Breaknews', CNA_Breaknews, int(num_news)),
+        ('LTN_Breaknews', LTN_Breaknews, int(num_news)),
+        ('ETtoday_Breaknews', ETtoday_Breaknews, int(num_news)),
+        ('SETN_Breaknews', SETN_Breaknews, int(num_news))
+    ]
+    for source_name, source_func, num_news in news_sources:
+        try:
+            all_news.extend(source_func(num_news))
+        except Exception as e:
+            print(f"Error fetching news from {source_name}: {e}")
+    total_news = len(all_news)
+    estimated_time = total_news * 7  # 每條新聞預估7秒
+    estimated_minutes = estimated_time // 60
+    estimated_seconds = estimated_time % 60
+    gr.Info(f"預估處理時間：約 {estimated_minutes} 分 {estimated_seconds} 秒")
+
+    client = OpenAI(
+        api_key=os.environ.get("YOUR_API_KEY"),
+        base_url="https://api.sambanova.ai/v1",
+    )
+    keywords = ["房屋稅","地價稅","牌照稅","土地增值稅","契稅","印花稅","娛樂稅","特別稅","囤房稅","財政部","財政資訊中心","財資中心","臺北市稅捐稽徵處","新北市政府稅捐稽徵處","桃園市政府地方稅務局","臺中市政府地方稅務局","臺南市政府財政稅務局","高雄市稅捐稽徵處","新竹市稅務局","新竹縣政府稅務局","苗栗縣政府稅務局","彰化縣地方稅務局","南投縣稅務局","雲林縣稅務局","嘉義市政府財政稅務局","嘉義縣財政稅務局","屏東縣政府財稅局","基隆市稅務局","宜蘭縣政府財政稅務局","花蓮縣地方稅務局","臺東縣稅務局","澎湖縣政府稅務局","金門縣稅務局","連江縣財政稅務局","生成式AI","人工智慧","AI智慧","AI模型","個資外洩","資安漏洞","隱私洩漏","稅務局","地方稅","稅捐機關","稅務入口網","地方稅網路申報"]
+    prompt = f"""
+    請判斷以下新聞內容與這些關鍵字的相關性：```{', '.join(keywords)}```
+    請給出一個0到100之間的相關性百分比，並說明理，理由說明不要提到keywords內容。
+    回答格式為：
+    相關性：XX%
+    理由：（簡要說明）
+
+    新聞內容：```
+    """
+    for i, (title, link, article) in enumerate(all_news):
+        full_prompt = prompt + article + "```"
+        max_retries = 2
+        retry_delay = 5
+        for attempt in range(max_retries):
+            try:
+                response = client.chat.completions.create(
+                    model="Meta-Llama-3.1-405B-Instruct",
+                    messages=[
+                        {"role": "system", "content": "Must reply to user in Traditional Chinese."},
+                        {"role": "user", "content": full_prompt}
+                    ],
+                    temperature=1
+                )
+                result = response.choices[0].message.content.strip()
+                break
+            except Exception as e:
+                if (attempt == 0) or (attempt == max_retries - 1):
+                    try:
+                        response = client.chat.completions.create(
+                            model="Meta-Llama-3.1-70B-Instruct",
+                            messages=[
+                                {"role": "system", "content": "Must reply to user in Traditional Chinese."},
+                                {"role": "user", "content": full_prompt}
+                            ],
+                            temperature=0.5
+                        )
+                        result = response.choices[0].message.content.strip()
+                        print("Successfully used 70B model as fallback")
+                        break
+                    except Exception as e2:
+                        print(f"Error with both models: {e2}")
+                        result = "無法處理此新聞項目"
+                        break
+                else:
+                    print(f"Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                    retry_delay *= 2
+        relevance_percentage = 0
+        reason = ""
+        for line in result.split('\n'):
+            if line.startswith("相關性："):
+                try:
+                    relevance_percentage = int(line.split("：")[1].strip().rstrip('%'))
+                except ValueError:
+                    print(f"Warning: Could not parse relevance percentage from '{line}'")
+                    relevance_percentage = 0
+            elif line.startswith("理由："):
+                reason = line.split("：")[1].strip()
+        relevance = "高" if relevance_percentage >= 60 else "中" if relevance_percentage >= 40 else "無"
+        all_news[i] = (title, link, article, f"Relevance: '{relevance}', Rate: {relevance_percentage}%, Reason: {reason}")
+        print(f"News item {i} processed: {all_news[i]}")
+        # 在每次API調用之間增加隨機延遲
+        time.sleep(random.uniform(2, 5))
+        if total_news > 10 and (i + 1) % (total_news // 5) == 0:
+            progress_percentage = ((i + 1) / total_news) * 100
+            gr.Info(f"已處理 {progress_percentage:.0f}% 的新聞項目")
+
+    news_html = """
+    <style>
+        .high-relevance:hover {
+            color: red !important;
+        }
+        .news-list {
+            list-style-type: disc;
+            padding-left: 10px;
+        }
+        .news-item {
+            margin-bottom: 15px;
+        }
+    </style>
+    <ul class="news-list">
+    """
+    high_relevance_news = []
+    for title, link, article, relevance_info in all_news:
+        relevance_parts = relevance_info.split(', ')
+        relevance = relevance_parts[0].split("'")[1]
+        rate = int(relevance_parts[1].split(': ')[1].rstrip('%'))
+        reason = relevance_parts[2].split(': ')[1]
+        if rate >= 40:
+            high_relevance_news.append((title, link, rate, reason))
+        if rate >= 60:
+            title_style = 'color: #C62E2E; font-weight: bold;'
+            title_class = 'class="high-relevance"'
+            info_color = '#C62E2E'
+        elif 40 <= rate < 60:
+            title_style = 'color: #F95454; font-weight: bold;'
+            title_class = 'class="high-relevance"'
+            info_color = 'black'
+        elif 20 <= rate < 40:
+            title_style = 'color: #FFA24C; font-weight: bold;'
+            title_class = 'class="high-relevance"'
+            info_color = 'black'
+        else:
+            title_style = 'color: blue;'
+            title_class = ''
+            info_color = 'black'
+        news_html += f"""
+        <li class="news-item">
+            <a href='{link}' target='_blank' style='text-decoration: none; {title_style}' {title_class}>{title}</a><br>
+            <span style='color: {info_color};'>LLM解析結果：{relevance}財稅相關，關聯度為 {rate} ％。</span><br>
+            <span style='color: {info_color};'>理由：{reason}</span>
+        </li>
+        """
+    news_html += "</ul>"
+    news_html += f"<p style='text-align: right; font-style: italic; color: #888;'>更新時間：{current_time}</p>"
+    if high_relevance_news:
+        resend.api_key = os.environ["YOUR_API_TOKEN"]
+        email_content = "<h2>財資焦點新聞摘要</h2><ul>"
+        for title, link, rate, reason in high_relevance_news:
+            email_content += f"<li><a href='{link}'>{title}</a>（相關性：{rate}%）<br>理由：{reason}</li>"
+        email_content += "</ul>"
+        params: resend.Emails.SendParams = {
+            "from": "News_Get <onboarding@resend.dev>",
+            "to": ["antivir7@gmail.com"],
+            "subject": "財資關鍵焦點新聞（閥值40%）",
+            "html": email_content,
+        }
+        try:
+            email_response = resend.Emails.send(params)
+            print(f"High relevance news email sent successfully. Response: {email_response}")
+        except Exception as e:
+            print(f"Failed to send high relevance news email: {str(e)}")
+    end_time = time.time()
+    gr.Info(f"焦點新聞已更新完成，執行時間： {(end_time - start_time):.2f} 秒。")
+    return news_html
+
+last_update_time = 0
+last_num_news = 0
+
+def update_news(num_news):
+    global last_update_time, last_num_news
+    current_time = time.time()
+    num_news = int(num_news)
+    if num_news <= 10 and last_num_news <= 20:
+        cooldown = 180
+    elif 10 <= num_news <= 20 and last_num_news <= 10:
+        cooldown = 180
+    elif 10 <= num_news <= 20 and 10 < last_num_news <= 20:
+        cooldown = 300
+    elif 10 <= num_news <= 20 and last_num_news <= 30:
+        cooldown = 600
+    elif 10 <= num_news <= 20 and last_num_news <= 40:
+        cooldown = 1200
+    else:
+        cooldown = 1800
+    if current_time - last_update_time < cooldown:
+        remaining_time = int(cooldown - (current_time - last_update_time))
+        gr.Warning(f"為避免遭誤判為DDoS，請等待 {remaining_time} 秒後再次更新")
+        if last_update_time == 0:
+            return gr.update(value=f"為避免遭誤判為DDoS，請等待 {remaining_time} 秒後再次更新"), gr.update(visible=True)
+        else:
+            return gr.update(), gr.update(visible=True)
+    last_update_time = current_time
+    last_num_news = num_news
+    resend.api_key = os.environ["YOUR_API_TOKEN"]
+    params: resend.Emails.SendParams = {
+        "from": "News_API <onboarding@resend.dev>",
+        "to": ["antivir7@gmail.com"],
+        "subject": "掌握焦點新聞",
+        "html": f"""
+        <strong>焦點新聞內容更新</strong><br>
+        每家獲取數量：{num_news}<br>
+        更新時間：{datetime.datetime.fromtimestamp(last_update_time).strftime('%Y-%m-%d %H:%M:%S')}
+        """,
+    }
+    try:
+        email_response = resend.Emails.send(params)
+        print(f"Email sent successfully. Response：{email_response}")
+    except Exception as e:
+        print(f"Failed to send email：{str(e)}")
+    news_html = initial_news(num_news)
+    return news_html, gr.update(visible=True)
+
+with gr.Blocks(theme=gr.themes.Monochrome(), css=custom_css) as iface:
+    gr.Markdown("""
+    # 🔥 最新焦點新聞 - 財政部財政資訊中心 🔥
+    > ### **※ 快速掌握關注之最新焦點新聞，為避免遭誤判為DDoS，請適當調整每家獲取數量。系統部署：江信宗，LLM：Llama-3.1-70B。**
+    """, elem_classes="center-aligned")
+    with gr.Row():
+        num_news = gr.Slider(
+            minimum=1,
+            maximum=40 if is_peak_hours() else 20,
+            value=5,
+            step=1,
+            label="每家獲取數量",
+            info="每家爬取新聞則數進行LLM解析",
+            elem_classes="file-background"
+        )
+        with gr.Column():
+            gr.HTML(
+            """
+            <span style="font-size: 18px; color: black;">為免相同新聞爬取過多，僅以5間新聞媒體作示範，取得新聞數量越多，LLM解析時間越長，敬請耐心等候。</span>
+            """, elem_classes="script-background"
+            )
+            update_button = gr.Button("掌握焦點新聞", elem_id="submit-btn")
+    opinion_news = gr.HTML(label="焦點新聞", elem_classes="text-background", visible=False)
+    update_button.click(fn=update_news, inputs=[num_news], outputs=[opinion_news, opinion_news])
+
+if __name__ == "__main__":
+    if "SPACE_ID" in os.environ:
+        iface.launch()
+    else:
+        iface.launch(share=True, show_api=False)

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+openai
+requests
+resend
+beautifulsoup4
+lxml
+pytz