app.py

import streamlit as st
import plotly.graph_objs as go
import requests
import json
import pandas as pd
import time
from pytrends.request import TrendReq
from pytrends.exceptions import TooManyRequestsError

# 設置 Streamlit 應用程序
st.title("PCHOME 和 MOMO 商品價格爬蟲分析與趨勢分析")

# 定義基礎URL
pchome_base_url = 'https://ecshweb.pchome.com.tw/search/v3.3/all/results?q='
momo_url = "https://apisearch.momoshop.com.tw/momoSearchCloud/moec/textSearch"

# 爬取 PCHOME 的函數
def crawl_pchome(keyword, num_pages):
    alldata = pd.DataFrame()
    for i in range(1, num_pages + 1):
        url = f'{pchome_base_url}{keyword}&page={i}&sort=sale/dc'
        list_req = requests.get(url)
        getdata = json.loads(list_req.content)
        todataFrame = pd.DataFrame(getdata['prods'])
        alldata = pd.concat([alldata, todataFrame])
        time.sleep(1)
    return alldata[["name", "price"]]

# 爬取 MOMO 的函數
def crawl_momo(keyword, num_pages):
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
    }
    product_list = []
    for page in range(1, num_pages + 1):
        payload = {
            "host": "momoshop",
            "flag": "searchEngine",
            "data": {
                "searchValue": keyword,
                "curPage": str(page),
                "priceS": "0",
                "priceE": "9999999",
                "searchType": "1"
            }
        }
        response = requests.post(momo_url, headers=headers, json=payload)
        if response.status_code == 200:
            data_from_api = response.json()
            products = data_from_api.get('rtnSearchData', {}).get('goodsInfoList', [])
            for product in products:
                name = product.get('goodsName', '')
                price = product.get('goodsPrice', '')
                price_str = str(price).split('(')[0].replace(',', '').replace('$', '')
                try:
                    product_price = float(price_str)
                except ValueError:
                    product_price = 0
                product_list.append({'name': name, 'price': product_price})
        time.sleep(1)
    return pd.DataFrame(product_list)

# 獲取 Google 趨勢數據的函數
def get_trends_data(keyword, start_date, end_date):
    pytrends = TrendReq(hl='en-US', tz=360)
    kw_list = [keyword]
    timeframe = f'{start_date} {end_date}'
    
    for _ in range(3):  # 重試最多3次
        try:
            pytrends.build_payload(kw_list, cat=0, timeframe=timeframe, geo='TW', gprop='')
            trends_data = pytrends.interest_over_time()
            if not trends_data.empty:
                trends_data = trends_data.reset_index()
                return trends_data
        except TooManyRequestsError:
            st.warning("Google 趨勢請求過多，正在等待重試...")
            time.sleep(60)  # 等待60秒後重試
            
    st.error("無法獲取 Google 趨勢數據，請稍後再試。")
    return None

# 輸入關鍵字和頁數
keyword = st.text_input("請輸入關鍵字:", "平板")
num_pages = st.number_input("請輸入要爬取的頁數:", min_value=1, max_value=100, value=1)

# 選擇 Pytrends 時間範圍
start_date = st.date_input("選擇開始日期:", value=pd.to_datetime("2023-01-01"))
end_date = st.date_input("選擇結束日期:", value=pd.to_datetime("2023-12-31"))

# 主爬取函數
if st.button("開始爬取"):
    start_time = time.time()
    
    # 爬取 PCHOME 資料
    st.subheader("爬取 PCHOME 資料")
    pchome_progress = st.progress(0)
    pchome_data = crawl_pchome(keyword, num_pages)
    pchome_progress.progress(100)
    
    # 爬取 MOMO 資料
    st.subheader("爬取 MOMO 資料")
    momo_progress = st.progress(0)
    momo_data = crawl_momo(keyword, num_pages)
    momo_progress.progress(100)
    
    # 檢查是否有數據
    if pchome_data.empty and momo_data.empty:
        st.error("查無商品")
    else:
        # 添加來源欄位
        pchome_data['source'] = 'PCHOME'
        momo_data['source'] = 'MOMO'
        
        # 合併並排序數據
        combined_data = pd.concat([pchome_data, momo_data])
        combined_data = combined_data.sort_values('price', ascending=False).reset_index(drop=True)
        
        # 顯示結果
        st.subheader("爬取結果:")
        st.write(combined_data)
        
        # 生成 CSV 檔案
        csv = combined_data.to_csv(index=False)
        st.download_button(
            label="下載 CSV 檔案",
            data=csv,
            file_name=f"{keyword}_price_data.csv",
            mime="text/csv",
        )

        # 繪製價格分布圖 - 圓餅圖
        st.subheader("價格分布圖 - 圓餅圖")
        combined_data['price_range'] = pd.cut(combined_data['price'], bins=range(0, int(combined_data['price'].max()) + 1000, 1000))
        price_range_counts = combined_data['price_range'].value_counts().sort_index()
        fig_pie = go.Figure(data=[go.Pie(labels=price_range_counts.index.astype(str), values=price_range_counts.values)])
        fig_pie.update_layout(
            title='價格分布 - 圓餅圖',
            width=800,  # 設定圖表寬度
            height=800  # 設定圖表高度
            )
        st.plotly_chart(fig_pie)

        # 繪製價格分布圖 - 折線圖
        st.subheader("價格分布圖 - 折線圖")
        fig_line = go.Figure(data=[go.Scatter(x=price_range_counts.index.astype(str), y=price_range_counts.values, mode='lines+markers')])
        fig_line.update_layout(title='價格分布 - 折線圖', xaxis_title='價格區間', yaxis_title='商品數量',
            width=800,  # 設定圖表寬度
            height=800  # 設定圖表高度
            )
        st.plotly_chart(fig_line)
        
        # 繪製價格分布圖 - 南丁格爾玫瑰圖
        st.subheader("價格分布圖 - 南丁格爾玫瑰圖")
        fig_rose = go.Figure(data=[go.Barpolar(r=price_range_counts.values, theta=price_range_counts.index.astype(str), marker=dict(color=price_range_counts.values, colorscale='Viridis'))])
        fig_rose.update_layout(title='價格分布 - 南丁格爾玫瑰圖',
            width=800,  # 設定圖表寬度
            height=800  # 設定圖表高度
            )
        st.plotly_chart(fig_rose)

        # 繪製價格分布圖 - 環形圖
        st.subheader("價格分布圖 - 環形圖")
        fig_donut = go.Figure(data=[go.Pie(labels=price_range_counts.index.astype(str), values=price_range_counts.values, hole=0.4)])
        fig_donut.update_layout(title='價格分布 - 環形圖',
            width=800,  # 設定圖表寬度
            height=800  # 設定圖表高度
            )
        st.plotly_chart(fig_donut)

        # 顯示統計數據
        st.subheader("統計數據")
        st.write(f"平均價格: {combined_data['price'].mean():.2f}")
        st.write(f"最高價格: {combined_data['price'].max():.2f}")
        st.write(f"最低價格: {combined_data['price'].min():.2f}")
    
    # 獲取並繪製 Google 趨勢數據
    trends_data = get_trends_data(keyword, start_date, end_date)
    if trends_data is not None:
        st.subheader("Google 搜尋趨勢")
        fig = go.Figure()
        fig.add_trace(go.Scatter(x=trends_data['date'], y=trends_data[keyword], mode='lines', name='趨勢指數'))
        fig.update_layout(title=f'Google 搜尋趨勢: {keyword}', xaxis_title='日期', yaxis_title='趨勢指數')
        st.plotly_chart(fig)
    else:
        st.warning("無法獲取 Google 趨勢數據")
    
    # 執行時間
    end_time = time.time()
    execution_time = end_time - start_time
    st.write(f"執行時間: {execution_time:.2f} 秒")