Create app.py

app.py

@@ -0,0 +1,568 @@
+import streamlit as st
+import requests
+import pandas as pd
+import time
+import json
+from urllib.parse import urlencode
+import resend
+from datetime import datetime
+import io
+
+# 頁面配置
+st.set_page_config(
+    page_title="MeetTaiwan 活動爬蟲系統",
+    page_icon="🎯",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+class MeetTaiwanAPIScraper:
+    def __init__(self):
+        self.base_url = "https://service.meettaiwan.com"
+        self.api_base = "https://service.meettaiwan.com/gpa/api/v2/events"
+
+        # 設定session
+        self.session = requests.Session()
+        self.session.headers.update({
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': 'application/json, text/plain, */*',
+            'Accept-Language': 'zh-TW,zh;q=0.9,en;q=0.8',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Connection': 'keep-alive',
+            'Referer': 'https://service.meettaiwan.com/gpa/zh/events/list;type=all',
+            'Origin': 'https://service.meettaiwan.com'
+        })
+
+    def get_events_by_page(self, page=1, page_size=10, event_type=None):
+        """調用API獲取指定頁面的活動資料"""
+        try:
+            # 構建API URL
+            params = {
+                'page': page,
+                'pageSize': page_size
+            }
+
+            if event_type:
+                params['type'] = event_type
+
+            # 嘗試不同的API端點
+            api_endpoints = [
+                f"{self.api_base}/",  # 主要API
+                f"{self.api_base}/tt-events"  # TT events API
+            ]
+
+            for api_url in api_endpoints:
+                try:
+                    response = self.session.get(api_url, params=params, timeout=30)
+
+                    if response.status_code == 200:
+                        try:
+                            data = response.json()
+
+                            # 檢查資料結構
+                            if isinstance(data, dict):
+                                # 常見的資料結構
+                                if 'data' in data:
+                                    events = data['data']
+                                elif 'items' in data:
+                                    events = data['items']
+                                elif 'results' in data:
+                                    events = data['results']
+                                elif 'events' in data:
+                                    events = data['events']
+                                else:
+                                    events = data
+
+                                if isinstance(events, list) and events:
+                                    return events, data
+                                else:
+                                    continue
+
+                            elif isinstance(data, list):
+                                return data, data
+
+                        except json.JSONDecodeError as e:
+                            continue
+
+                except Exception as e:
+                    continue
+
+            return None, None
+
+        except Exception as e:
+            return None, None
+
+    def get_all_events(self, progress_callback=None):
+        """獲取所有類型的活動資料"""
+        all_events = []
+        page_size_options = [50, 30, 20]
+        max_pages = 20
+
+        for page_size in page_size_options:
+            for page in range(1, max_pages + 1):
+                if progress_callback:
+                    progress_callback(f"正在獲取第 {page} 頁資料 (頁面大小: {page_size})")
+
+                events, raw_data = self.get_events_by_page(page=page, page_size=page_size, event_type=None)
+
+                if not events or len(events) == 0:
+                    if page == 1:
+                        break
+                    else:
+                        return all_events
+
+                # 處理事件資料
+                for event in events:
+                    processed_event = self.process_event_data(event, page, 'All')
+                    if processed_event:
+                        # 檢查是否已存在（避免重複）
+                        is_duplicate = False
+                        for existing_event in all_events:
+                            if (existing_event['name'] == processed_event['name'] and
+                                existing_event['event_date'] == processed_event['event_date']):
+                                is_duplicate = True
+                                break
+
+                        if not is_duplicate:
+                            all_events.append(processed_event)
+
+                # 如果這一頁的資料少於頁面大小，可能是最後一頁
+                if len(events) < page_size:
+                    return all_events
+
+                time.sleep(0.5)  # 減少延遲
+
+            # 如果成功獲取到資料，就��需要嘗試其他頁面大小
+            if all_events:
+                break
+
+        return all_events
+
+    def process_event_data(self, event, page_num, event_type):
+        """處理單個活動資料"""
+        try:
+            if isinstance(event, dict):
+                name = event.get('name', event.get('title', event.get('eventName', '')))
+                form = event.get('type', event.get('category', event.get('eventType', event_type or '')))
+                event_date = event.get('eventDate', event.get('startDate', event.get('date', '')))
+                upload_date = event.get('createdAt', event.get('uploadDate', event.get('publishDate', '')))
+
+                # 構建連結
+                event_id = event.get('id', event.get('eventId', ''))
+                if event_id:
+                    link = f"{self.base_url}/gpa/zh/events/{form}/{event_id}"
+                else:
+                    link = ""
+
+                return {
+                    'name': str(name),
+                    'link': link,
+                    'form': str(form),
+                    'event_date': str(event_date),
+                    'upload_date': str(upload_date),
+                    'page_num': page_num
+                }
+
+        except Exception as e:
+            pass
+
+        return None
+
+def create_html_table(df, max_display=10):
+    """將活動資料轉換為 HTML 表格格式"""
+    if df is None or df.empty:
+        return "<p>沒有找到活動資料</p>"
+
+    display_count = min(len(df), max_display)
+    df_display = df.head(display_count)
+    
+    # 創建 HTML 表格
+    html_content = f"""
+    <html>
+    <head>
+        <meta charset="UTF-8">
+        <style>
+            body {{ font-family: Arial, sans-serif; margin: 20px; line-height: 1.6; }}
+            .greeting {{ 
+                font-size: 18px; 
+                color: #2c3e50; 
+                margin-bottom: 20px; 
+                font-weight: bold;
+            }}
+            h2 {{ color: #2c3e50; margin-top: 20px; }}
+            .data-source {{
+                background-color: #e8f4f8;
+                padding: 15px;
+                border-left: 4px solid #3498db;
+                margin: 20px 0;
+                font-weight: bold;
+                color: #2c3e50;
+            }}
+            table {{ 
+                border-collapse: collapse; 
+                width: 100%; 
+                margin-top: 20px;
+                box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+            }}
+            th, td {{ 
+                border: 1px solid #ddd; 
+                padding: 12px; 
+                text-align: left; 
+            }}
+            th {{ 
+                background-color: #3498db; 
+                color: white;
+                font-weight: bold;
+            }}
+            tr:nth-child(even) {{ background-color: #f2f2f2; }}
+            tr:hover {{ background-color: #e8f4f8; }}
+            a {{ color: #3498db; text-decoration: none; }}
+            a:hover {{ text-decoration: underline; }}
+            .summary {{ 
+                background-color: #ecf0f1; 
+                padding: 15px; 
+                border-radius: 5px; 
+                margin-bottom: 20px;
+            }}
+            .copyright {{
+                text-align: center;
+                margin-top: 30px;
+                padding: 20px;
+                background-color: #34495e;
+                color: white;
+                border-radius: 5px;
+                font-size: 14px;
+            }}
+        </style>
+    </head>
+    <body>
+        <div class="greeting">
+            親愛的會員您好：
+        </div>
+        
+        <div class="data-source">
+            📋 資料來源：全球政府採購商機網
+        </div>
+        
+        <h2>🎯 最新活動資訊</h2>
+        <div class="summary">
+            <strong>📊 資料統計：</strong>顯示前 {display_count} 筆，共 {len(df)} 筆活動
+        </div>
+        <table>
+            <thead>
+                <tr>
+                    <th>序號</th>
+                    <th>名稱</th>
+                    <th>形式</th>
+                    <th>活動日期</th>
+                    <th>上載日期</th>
+                    <th>網址</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
+    for idx, row in df_display.iterrows():
+        link_html = f'<a href="{row["超連結網址"]}" target="_blank">查看詳情</a>' if row["超連結網址"] else "無連結"
+        html_content += f"""
+                <tr>
+                    <td>{idx + 1}</td>
+                    <td><strong>{row['名稱']}</strong></td>
+                    <td>{row['形式']}</td>
+                    <td>{row['活動日期']}</td>
+                    <td>{row['上載日期']}</td>
+                    <td>{link_html}</td>
+                </tr>
+        """
+    
+    html_content += """
+            </tbody>
+        </table>
+    """
+    
+    if len(df) > max_display:
+        html_content += f"""
+        <div class="summary" style="margin-top: 20px;">
+            <strong>📝 提醒：</strong>還有 {len(df) - max_display} 筆資料未顯示，
+            請查看附加的 CSV 檔案獲取完整資料。
+        </div>
+        """
+    
+    html_content += """
+        <div class="summary" style="margin-top: 20px;">
+            <strong>🤖 自動爬蟲系統</strong><br>
+            此郵件由 MeetTaiwan API 爬蟲系統自動產生並發送
+        </div>
+        
+        <div class="copyright">
+            2025 © Copyright robert_studio
+        </div>
+    </body>
+    </html>
+    """
+    
+    return html_content
+
+def send_events_email(df_events, recipient_email, api_key, max_display=5):
+    """發送活動資料到指定郵箱"""
+    if df_events is None or df_events.empty:
+        return False, "沒有資料可發送"
+    
+    try:
+        # 設定 Resend API Key
+        resend.api_key = api_key
+        
+        # 取前N筆資料用於顯示
+        df_display = df_events.head(max_display)
+        
+        # 建立 HTML 內容
+        html_content = create_html_table(df_display, max_display=max_display)
+        
+        # 準備郵件主題
+        current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        subject = f"📊 MeetTaiwan 最新活動資訊 - {len(df_events)}筆活動 ({current_time})"
+        
+        # 發送郵件
+        r = resend.Emails.send({
+            "from": "onboarding@resend.dev",
+            "to": recipient_email,
+            "subject": subject,
+            "html": html_content
+        })
+        
+        return True, f"郵件發送成功！郵件 ID: {r.get('id', 'N/A')}"
+        
+    except Exception as e:
+        return False, f"郵件發送失敗: {str(e)}"
+
+# Streamlit 應用程式主體
+def main():
+    # 標題和說明
+    st.title("🎯 MeetTaiwan 活動爬蟲系統")
+    st.markdown("**全球政府採購商機網活動資訊自動抓取與郵件發送系統**")
+    
+    # 側邊欄設定
+    st.sidebar.header("⚙️ 系統設定")
+    
+    # API Key 設定
+    api_key = st.sidebar.text_input(
+        "Resend API Key",
+        value="re_ZGacBiDw_HFEBpuCbaJ2S3NThPWiMU7Ex",
+        type="password",
+        help="請輸入您的 Resend API Key"
+    )
+    
+    # 收件人設定
+    recipient_email = st.sidebar.text_input(
+        "收件人郵箱",
+        value="cjhuang38@gmail.com",
+        help="請輸入要接收報告的郵箱地址"
+    )
+    
+    # 顯示筆數設定
+    max_display = st.sidebar.slider(
+        "郵件顯示筆數",
+        min_value=5,
+        max_value=20,
+        value=10,
+        help="設定郵件中要顯示的活動筆數"
+    )
+    
+    # 初始化 session state
+    if 'df_events' not in st.session_state:
+        st.session_state.df_events = None
+    if 'scraping_done' not in st.session_state:
+        st.session_state.scraping_done = False
+    
+    # 主要操作區域
+    col1, col2 = st.columns([2, 1])
+    
+    with col1:
+        st.header("📊 資料爬取")
+        
+        if st.button("🚀 開始爬取活動資料", type="primary", use_container_width=True):
+            # 清除之前的資料
+            st.session_state.df_events = None
+            st.session_state.scraping_done = False
+            
+            # 建立進度條和狀態顯示
+            progress_bar = st.progress(0)
+            status_text = st.empty()
+            log_container = st.empty()
+            
+            # 建立爬蟲實例
+            scraper = MeetTaiwanAPIScraper()
+            
+            def update_progress(message):
+                status_text.text(f"🔄 {message}")
+            
+            try:
+                # 開始爬取
+                with st.spinner("正在爬取資料..."):
+                    events_data = scraper.get_all_events(progress_callback=update_progress)
+                
+                if events_data and len(events_data) > 0:
+                    # 轉換為 DataFrame
+                    df_events = pd.DataFrame(events_data)
+                    df_events.columns = ["名稱", "超連結網址", "形式", "活動日期", "上載日期", "頁數"]
+                    
+                    # 去除重複資料
+                    original_count = len(df_events)
+                    df_events = df_events.drop_duplicates(subset=['名稱', '活動日期'])
+                    deduplicated_count = len(df_events)
+                    
+                    # 儲存到 session state
+                    st.session_state.df_events = df_events
+                    st.session_state.scraping_done = True
+                    
+                    # 更新進度條
+                    progress_bar.progress(100)
+                    status_text.text(f"✅ 爬取完成！獲得 {deduplicated_count} 筆活動資料")
+                    
+                    if original_count != deduplicated_count:
+                        st.info(f"📝 去除了 {original_count - deduplicated_count} 筆重複資料")
+                    
+                    st.success(f"🎉 成功獲取 {len(df_events)} 筆活動資料！")
+                
+                else:
+                    status_text.text("❌ 無法獲取活動資料")
+                    st.error("爬取失敗，可能原因：API端點變更、需要認證或網路問題")
+            
+            except Exception as e:
+                status_text.text(f"❌ 爬取過程發生錯誤: {str(e)}")
+                st.error(f"執行錯誤：{str(e)}")
+    
+    with col2:
+        st.header("📧 郵件發送")
+        
+        if st.session_state.df_events is not None:
+            st.success(f"📊 已載入 {len(st.session_state.df_events)} 筆資料")
+            
+            if st.button("📨 發送郵件報告", type="secondary", use_container_width=True):
+                if not api_key:
+                    st.error("請輸入 Resend API Key")
+                elif not recipient_email:
+                    st.error("請輸入收件人郵箱")
+                else:
+                    with st.spinner("正在發送郵件..."):
+                        success, message = send_events_email(
+                            st.session_state.df_events, 
+                            recipient_email, 
+                            api_key, 
+                            max_display
+                        )
+                    
+                    if success:
+                        st.success(f"✅ {message}")
+                    else:
+                        st.error(f"❌ {message}")
+        else:
+            st.info("請先爬取資料")
+    
+    # 資料預覽區域
+    if st.session_state.df_events is not None:
+        st.header("📋 資料預覽")
+        
+        # 統計資訊
+        col1, col2, col3, col4 = st.columns(4)
+        with col1:
+            st.metric("總活動數", len(st.session_state.df_events))
+        with col2:
+            form_counts = st.session_state.df_events['形式'].nunique()
+            st.metric("活動形式數", form_counts)
+        with col3:
+            try:
+                year_counts = pd.to_datetime(st.session_state.df_events['活動日期'], errors='coerce').dt.year.nunique()
+                st.metric("涵蓋年份", year_counts)
+            except:
+                st.metric("涵蓋年份", "N/A")
+        with col4:
+            page_counts = st.session_state.df_events['頁數'].nunique()
+            st.metric("來源頁面數", page_counts)
+        
+        # 資料表格
+        st.subheader("📊 詳細資料")
+        
+        # 篩選選項
+        col1, col2 = st.columns(2)
+        with col1:
+            form_options = ['全部'] + list(st.session_state.df_events['形式'].unique())
+            selected_form = st.selectbox("選擇活動形式", form_options)
+        
+        with col2:
+            display_count = st.slider("顯示筆數", 10, min(100, len(st.session_state.df_events)), 20)
+        
+        # 篩選資料
+        if selected_form != '全部':
+            filtered_df = st.session_state.df_events[st.session_state.df_events['形式'] == selected_form]
+        else:
+            filtered_df = st.session_state.df_events
+        
+        # 顯示表格
+        st.dataframe(
+            filtered_df.head(display_count),
+            use_container_width=True,
+            hide_index=True
+        )
+        
+        # 下載功能
+        st.subheader("💾 資料下載")
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            # CSV 下載
+            csv = st.session_state.df_events.to_csv(index=False, encoding='utf-8-sig')
+            st.download_button(
+                label="📄 下載 CSV 檔案",
+                data=csv,
+                file_name=f"meettaiwan_events_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",
+                mime="text/csv",
+                use_container_width=True
+            )
+        
+        with col2:
+            # Excel 下載
+            buffer = io.BytesIO()
+            with pd.ExcelWriter(buffer, engine='openpyxl') as writer:
+                st.session_state.df_events.to_excel(writer, index=False, sheet_name='活動資料')
+            
+            st.download_button(
+                label="📊 下載 Excel 檔案",
+                data=buffer.getvalue(),
+                file_name=f"meettaiwan_events_{datetime.now().strftime('%Y%m%d_%H%M%S')}.xlsx",
+                mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                use_container_width=True
+            )
+        
+        # 統計圖表
+        st.subheader("📈 資料統計")
+        
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            # 活動形式分布
+            form_counts = st.session_state.df_events['形式'].value_counts()
+            st.bar_chart(form_counts, use_container_width=True)
+            st.caption("活動形式分布")
+        
+        with col2:
+            # 按頁面分布
+            if '頁數' in st.session_state.df_events.columns:
+                page_counts = st.session_state.df_events['頁數'].value_counts().sort_index()
+                st.bar_chart(page_counts, use_container_width=True)
+                st.caption("各頁面資料分布")
+    
+    # 頁尾資訊
+    st.markdown("---")
+    st.markdown(
+        """
+        <div style='text-align: center; color: #666;'>
+            <p>🤖 MeetTaiwan API 爬蟲系統 | 2025 © Copyright robert_studio</p>
+            <p>資料來源：全球政府採購商機網</p>
+        </div>
+        """, 
+        unsafe_allow_html=True
+    )
+
+if __name__ == "__main__":
+    main()