app.py

import requests
from bs4 import BeautifulSoup
import pandas as pd
import jieba
from keybert import KeyBERT
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import streamlit as st
import matplotlib.pyplot as plt
from matplotlib.font_manager import FontProperties
from wordcloud import WordCloud

# 下載字體
def download_font(url, save_path):
    response = requests.get(url)
    with open(save_path, 'wb') as f:
        f.write(response.content)

# 字體URL和保存路徑
font_url = 'https://drive.google.com/uc?id=1eGAsTN1HBpJAkeVM57_C7ccp7hbgSz3_&export=download'
font_path = 'TaipeiSansTCBeta-Regular.ttf'

# 下載字體
download_font(font_url, font_path)

# 設置字體
font_prop = FontProperties(fname=font_path)

# 抓取Yahoo新聞標題和內容
def fetch_yahoo_news(url):
    response = requests.get(url)
    web_content = response.content
    soup = BeautifulSoup(web_content, 'html.parser')
    title = soup.find('h1').text
    content = soup.find('article').text
    return title, content

# 斷詞函數
def jieba_tokenizer(text):
    return jieba.lcut(text)

# 初始化CountVectorizer並定義KeyBERT模型
vectorizer = CountVectorizer(tokenizer=jieba_tokenizer)
kw_model = KeyBERT()

# 提取關鍵詞的函數
def extract_keywords(doc, diversity=0.5):
    keywords = kw_model.extract_keywords(doc, vectorizer=vectorizer, use_mmr=True, diversity=diversity)
    return keywords

# 畫關鍵詞圖函數
def plot_keywords(keywords, title):
    words = [kw[0] for kw in keywords]
    scores = [kw[1] for kw in keywords]

    plt.figure(figsize=(10, 6))
    bars = plt.barh(words, scores, color='skyblue', edgecolor='black', linewidth=1.2)
    plt.xlabel('分數', fontproperties=font_prop, fontsize=14)
    plt.title(title, fontproperties=font_prop, fontsize=16)
    plt.gca().invert_yaxis()  # 反轉Y軸，使得分數最高的關鍵詞在最上面
    plt.xticks(fontproperties=font_prop, fontsize=12)
    plt.yticks(fontproperties=font_prop, fontsize=12)
    plt.grid(axis='x', linestyle='--', alpha=0.7)
    
    # 添加分數標籤
    for bar in bars:
        plt.gca().text(bar.get_width() + 0.01, bar.get_y() + bar.get_height() / 2,
                       f'{bar.get_width():.4f}', va='center', ha='left', fontsize=12, fontproperties=font_prop)

    st.pyplot(plt)

# 生成文字雲函數
def generate_wordcloud(text, title):
    wordcloud = WordCloud(font_path=font_path, background_color='white', width=800, height=400, max_words=200).generate(text)
    plt.figure(figsize=(10, 6))
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.title(title, fontproperties=font_prop, fontsize=16)
    plt.axis('off')
    st.pyplot(plt)

# Fit and display LDA topics
def display_topics(model, feature_names, num_top_words, title):
    topics = []
    for topic_idx, topic in enumerate(model.components_):
        topics.append(" ".join([feature_names[i] for i in topic.argsort()[:-num_top_words - 1:-1]]))

    plt.figure(figsize=(10, 6))
    plt.barh(range(len(topics)), model.components_.sum(axis=1), color='skyblue', edgecolor='black', linewidth=1.2)
    plt.xlabel('權重總和', fontproperties=font_prop, fontsize=14)
    plt.title(title, fontproperties=font_prop, fontsize=16)
    plt.gca().invert_yaxis()  # 反轉Y軸
    plt.xticks(fontproperties=font_prop, fontsize=12)
    plt.yticks(range(len(topics)), topics, fontproperties=font_prop, fontsize=12)
    plt.grid(axis='x', linestyle='--', alpha=0.7)

    st.pyplot(plt)

# 建立Streamlit網頁應用程式
st.title("👾⚓🌠YAHOO新聞關鍵詞提取工具👾⚓🌠")

# 抓取Yahoo新聞的URL輸入框
url = st.text_input("請輸入Yahoo新聞的URL：")

# MMR參數輸入框
diversity = st.slider("選擇MMR多樣性參數（0到1之間）：", 0.0, 1.0, 0.5)

# Number of LDA topics
num_topics = st.slider("選擇LDA主題數量：", 1, 10, 5)
num_top_words = st.slider("選擇每個主題顯示的單詞數量：", 1, 20, 10)

if st.button("抓取並提取關鍵詞"):
    if url:
        title, content = fetch_yahoo_news(url)
        st.write("新聞標題：", title)
        st.write("新聞內容：", content)
        
        # 將內容轉為DataFrame
        data = {'Title': [title], 'Content': [content]}
        df = pd.DataFrame(data)
        st.write("新聞內容的DataFrame：")
        st.write(df)
        
        # 提取關鍵詞
        keywords = extract_keywords(content, diversity)
        st.write("關鍵詞提取結果：")
        for keyword in keywords:
            st.write(f"{keyword[0]}: {keyword[1]:.4f}")
        
        plot_keywords(keywords, "關鍵詞提取結果")

        # 使用另一個模型進行關鍵詞提取
        kw_model_multilingual = KeyBERT(model='distiluse-base-multilingual-cased-v1')
        keywords_multilingual = kw_model_multilingual.extract_keywords(content, vectorizer=vectorizer, use_mmr=True, diversity=diversity)
        st.write("多語言模型關鍵詞提取結果：")
        for keyword in keywords_multilingual:
            st.write(f"{keyword[0]}: {keyword[1]:.4f}")
        
        plot_keywords(keywords_multilingual, "多語言模型關鍵詞提取結果")

        # 生成TF-IDF文字雲
        tfidf_vectorizer = TfidfVectorizer(tokenizer=jieba_tokenizer)
        tfidf_matrix = tfidf_vectorizer.fit_transform([content])
        tfidf_feature_names = tfidf_vectorizer.get_feature_names_out()
        tfidf_scores = tfidf_matrix.toarray().flatten()
        tfidf_dict = dict(zip(tfidf_feature_names, tfidf_scores))
        
        tfidf_text = ' '.join([f'{word} ' * int(score * 100) for word, score in tfidf_dict.items()])
        generate_wordcloud(tfidf_text, "TF-IDF文字雲")

        # LDA topic modeling
        lda_vectorizer = CountVectorizer(tokenizer=jieba_tokenizer)
        lda_matrix = lda_vectorizer.fit_transform([content])
        lda = LatentDirichletAllocation(n_components=num_topics, random_state=0)
        lda.fit(lda_matrix)

        display_topics(lda, lda_vectorizer.get_feature_names_out(), num_top_words, "LDA主題模型")

    else:
        st.write("請輸入有效的Yahoo新聞URL。")